Merge pull request #22 from openownership/register-fixes

Register fixes

.env.example

@@ -2,7 +2,10 @@
 #DC_WEB_LIB_REGISTER_SOURCES_BODS=../register-sources-bods
 #DC_WEB_LIB_REGISTER_SOURCES_OC=../register-sources-oc
 #DC_WEB_LIB_REGISTER_SOURCES_PSC=../register-sources-psc
+#DC_WEB_LIB_REGISTER_SOURCES_SK=../register-sources-sk
+#DC_WEB_LIB_REGISTER_SOURCES_DK=../register-sources-dk
 ADMIN_EMAILS=
+BODS_EXPORT_AWS_REGION=eu-west-1
 BODS_EXPORT_AWS_ACCESS_KEY_ID=
 BODS_EXPORT_AWS_SECRET_ACCESS_KEY=
 BODS_EXPORT_S3_BUCKET_NAME=

.env.test

@@ -4,6 +4,7 @@ SITE_BASE_URL="http://0.0.0.0:3000"
 REDIS_URL="redis://localhost:6379/1"
 BODS_EXPORT_S3_BUCKET_NAME=test-export-bucket
 BODS_EXPORT_AWS_ACCESS_KEY_ID=test
+BODS_EXPORT_AWS_REGION=eu-west-1
 BODS_EXPORT_AWS_SECRET_ACCESS_KEY=test
 DEV_DATA_AWS_ACCESS_KEY_ID=test
 DEV_DATA_AWS_SECRET_ACCESS_KEY=test

Gemfile

@@ -47,6 +47,8 @@ gem 'register_common', git: 'https://github.com/openownership/register-common.gi
 gem 'register_sources_oc', git: 'https://github.com/openownership/register-sources-oc.git'
 gem 'register_sources_psc', git: 'https://github.com/openownership/register-sources-psc.git'
 gem 'register_sources_bods', git: 'https://github.com/openownership/register-sources-bods.git'
+gem 'register_sources_sk', git: 'https://github.com/openownership/register-sources-sk.git'
+gem 'register_sources_dk', git: 'https://github.com/openownership/register-sources-dk.git'
 
 group :development, :test do
   gem 'byebug', '~> 11.1'

Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 
 GIT
   remote: https://github.com/openownership/register-sources-bods.git
-  revision: 9952fbb644b870ecb13899e03bb130762526bf6a
+  revision: 9fc7ec354ac58480556d0a44d581e9bffd4886a5
   specs:
     register_sources_bods (0.1.0)
       activesupport (>= 6, < 8)
@@ -30,6 +30,19 @@ GIT
       iso8601
       xxhash
 
+GIT
+  remote: https://github.com/openownership/register-sources-dk.git
+  revision: b97deb65255d825e11c94108785744936f3abec6
+  specs:
+    register_sources_dk (0.1.0)
+      activesupport (>= 6, < 8)
+      countries (~> 4.0.1)
+      dry-struct (>= 1, < 2)
+      dry-types (>= 1, < 2)
+      elasticsearch (>= 7.10, < 8)
+      iso8601
+      xxhash
+
 GIT
   remote: https://github.com/openownership/register-sources-oc.git
   revision: 04ef5f2991164717d90f9f0499d082c296335e89
@@ -50,7 +63,7 @@ GIT
 
 GIT
   remote: https://github.com/openownership/register-sources-psc.git
-  revision: 7dcd200bddca4c99497d0676a9fd5bcacf807939
+  revision: c9755e8e37bc095b2e580c2c14f9d65c5f33f5cc
   specs:
     register_sources_psc (0.1.0)
       activesupport (>= 6, < 8)
@@ -61,6 +74,19 @@ GIT
       iso8601
       xxhash
 
+GIT
+  remote: https://github.com/openownership/register-sources-sk.git
+  revision: b96fbdaa1d9bb7103271b616fa1ba74582b44fe4
+  specs:
+    register_sources_sk (0.1.0)
+      activesupport (>= 6, < 8)
+      countries (~> 4.0.1)
+      dry-struct (>= 1, < 2)
+      dry-types (>= 1, < 2)
+      elasticsearch (>= 7.10, < 8)
+      iso8601
+      xxhash
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -566,8 +592,10 @@ DEPENDENCIES
   redcarpet (~> 3.5)
   register_common!
   register_sources_bods!
+  register_sources_dk!
   register_sources_oc!
   register_sources_psc!
+  register_sources_sk!
   rexml (~> 3.2)
   roadie-rails (~> 2.3)
   rollbar (~> 3.3)

app.json

@@ -17,6 +17,9 @@
     "BODS_EXPORT_AWS_SECRET_ACCESS_KEY": {
       "required": true
     },
+    "BODS_EXPORT_AWS_REGION": {
+      "required": true
+    },
     "BODS_EXPORT_S3_BUCKET_NAME": {
       "required": true
     },

app/controllers/entities_controller.rb

@@ -76,10 +76,12 @@ def show
       @similar_people = entity.natural_person? ? similar_people(entity) : nil
     end
 
-    @data_source_names = DATA_SOURCE_REPOSITORY.data_source_names_for_entity(entity)
+    raw_records = RAW_DATA_RECORD_REPOSITORY.all_for_entity(entity)
+    @data_source_names = DATA_SOURCE_REPOSITORY.data_source_names_for_raw_records(raw_records)
+
     unless @data_source_names.empty?
-      @newest_raw_record = RAW_DATA_RECORD_REPOSITORY.newest_for_entity(entity).data.notified_on # .updated_at
-      @raw_record_count = RAW_DATA_RECORD_REPOSITORY.all_for_entity(entity).size
+      @newest_raw_record = RAW_DATA_RECORD_REPOSITORY.newest_for_entity_date(entity)
+      @raw_record_count = raw_records.size
     end
 
     # Conversion
@@ -103,6 +105,8 @@ def show
           entity.merged_entities.map(&:bods_statement)
         ].compact.flatten.uniq { |s| s.statementID }
 
+        statements = BodsStatementSorter.new.sort_statements(statements)
+
         render json: JSON.pretty_generate(statements.as_json)
       end
     end
@@ -125,13 +129,13 @@ def raw
     end
     redirect_to_master_entity?(:raw, entity) && return
     @sentity = entity
-    @raw_data_records = RAW_DATA_RECORD_REPOSITORY.all_for_entity(entity) # .page(params[:page]).per(10)
+    @raw_data_records = RAW_DATA_RECORD_REPOSITORY.all_for_entity(entity)
     return if @raw_data_records.empty?
 
     @oc_data = get_opencorporates_company_hash(entity) || {}
-    @newest = RAW_DATA_RECORD_REPOSITORY.newest_for_entity(entity).data.notified_on # .updated_at
-    @oldest = RAW_DATA_RECORD_REPOSITORY.oldest_for_entity(entity).data.notified_on # created_at
-    @data_sources = DATA_SOURCE_REPOSITORY.all_for_entity(entity)
+    @newest = RAW_DATA_RECORD_REPOSITORY.newest_for_entity_date(entity)
+    @oldest = RAW_DATA_RECORD_REPOSITORY.oldest_for_entity_date(entity)
+    @data_sources = DATA_SOURCE_REPOSITORY.all_for_raw_records(@raw_data_records)
   end
 
   def opencorporates_additional_info

app/controllers/relationships_controller.rb

@@ -2,8 +2,8 @@ class RelationshipsController < ApplicationController
   ENTITY_SERVICE = Rails.application.config.entity_service
 
   def show
-    target_entity = ENTITY_SERVICE.find(params[:entity_id])
-    source_entity = resolve_master_entity(ENTITY_SERVICE.find(params[:id]))
+    target_entity = ENTITY_SERVICE.find_by_entity_id(params[:entity_id])
+    source_entity = resolve_master_entity(ENTITY_SERVICE.find_by_entity_id(params[:id]))
 
     relationships = InferredRelationshipGraph2
       .new(target_entity)

app/decorators/entity_graph_decorator.rb

@@ -42,7 +42,7 @@ def cytoscape_entity_node(node)
     entity = node.entity #.decorate(context: context)
     classes = entity.dissolution_date ? ['dissolved'] : []
     tooltip = nil
-    unless false # entity.is_a? UnknownPersonsEntity
+    unless entity.unknown?
       tooltip = h.render(
         partial: 'entities/graph_tooltip',
         locals: { entity: entity },

app/helpers/entity_helper.rb

@@ -1,6 +1,6 @@
 module EntityHelper
   def entity_link(entity, &block)
-    if entity.master_entity.present? #entity.is_a?(CircularOwnershipEntity) || entity.is_a?(UnknownPersonsEntity) || entity.master_entity.present?
+    if entity.master_entity.present? || entity.unknown? #entity.is_a?(CircularOwnershipEntity)
       capture(&block)
     else
       link_to(entity_path(entity.id), &block)

app/repositories/bods_export_repository.rb

@@ -4,7 +4,7 @@ class BodsExportRepository
   def initialize(s3_adapter: nil, s3_bucket: nil, s3_prefix: nil)
     @s3_adapter = s3_adapter || Rails.application.config.s3_adapter
     @s3_bucket = s3_bucket || ENV.fetch('BODS_EXPORT_S3_BUCKET_NAME')
-    @s3_prefix = s3_prefix || ENV.fetch('BODS_EXPORT_S3_PREFIX', 'public/exports/')
+    @s3_prefix = s3_prefix || ENV.fetch('BODS_EXPORT_S3_PREFIX', 'exports/')
   end
 
   def completed_exports(limit: 5)
@@ -20,11 +20,13 @@ def most_recent
   attr_reader :s3_adapter, :s3_bucket, :s3_prefix
 
   def list_all
-    s3_paths = s3_adapter.list_objects(s3_bucket: s3_bucket, s3_prefix: File.join(s3_prefix, 'statements.'))
+    s3_paths = s3_adapter.list_objects(s3_bucket: s3_bucket, s3_prefix: s3_prefix)
 
     s3_paths.sort.reverse.map do |s3_path|
+      matched = /ex(?<year>\d{4})(?<month>\d{2})(?<day>\d{2})/.match s3_path
+      next unless matched
       time = begin
-        Time.parse(s3_path.split('.')[1])
+        Time.new(matched[:year], matched[:month], matched[:day])
       rescue ArgumentError
         next
       end

app/repositories/data_source_repository.rb

@@ -1,3 +1,7 @@
+require 'register_sources_psc/structs/company_record'
+require 'register_sources_dk/structs/record'
+require 'register_sources_sk/structs/record'
+
 class DataSourceRepository
   def all
     path = File.join(File.dirname(__FILE__), 'datasources.json')
@@ -13,19 +17,29 @@ def find(id)
   end
 
   def find_many(ids)
-    all.filter { |data_source| ids.include? data_source.id }
+    all.filter { |data_source| ids.include?(data_source.id) || ids.include?(data_source.slug)  }
   end
 
   def where_overview_present
     all.filter { |data_source| data_source.overview.present? }
   end
 
-  def data_source_names_for_entity(entity)
-    ["UK PSC Register"] # TODO: generate from sources of entity (or identifiers)
+  def data_source_names_for_raw_records(raw_records)
+    datasource_names = raw_records.map do |raw_record|
+      case raw_record
+      when RegisterSourcesDk::Deltagerperson
+        "Denmark Central Business Register (Centrale Virksomhedsregister [CVR])"
+      when RegisterSourcesPsc::CompanyRecord
+        "UK PSC Register"
+      when RegisterSourcesSk::Record
+        "Slovakia Public Sector Partners Register (Register partnerov verejného sektora)"
+      end
+    end.compact.uniq.sort
   end
 
-  def all_for_entity(entity)
-    data_source_names = data_source_names_for_entity(entity)
-    all.filter { |data_source| data_source_names.include? data_source.name }
+  def all_for_raw_records(raw_records)
+    datasource_names = data_source_names_for_raw_records(raw_records)
+
+    all.filter { |data_source| datasource_names.include? data_source.name }
   end
 end