From 3148ced142ef1e645bde24425c2a73d06d820a61 Mon Sep 17 00:00:00 2001 From: Josh Williams Date: Thu, 10 Aug 2023 16:36:09 +0000 Subject: [PATCH] Removed obsolete datasources from datasources.json file --- Gemfile | 2 +- Gemfile.lock | 32 +- app/repositories/datasources.json | 551 ------------------------------ 3 files changed, 31 insertions(+), 554 deletions(-) diff --git a/Gemfile b/Gemfile index 3bef3a19..d8b7eb01 100644 --- a/Gemfile +++ b/Gemfile @@ -46,7 +46,7 @@ gem 'hashie', '~> 3.4', '>= 3.4.4' gem 'register_common', git: 'https://github.com/openownership/register-common.git' gem 'register_sources_oc', git: 'https://github.com/openownership/register-sources-oc.git' gem 'register_sources_psc', git: 'https://github.com/openownership/register-sources-psc.git' -gem 'register_sources_bods', git: 'https://github.com/openownership/register-sources-bods.git', branch: 'register-fixes' +gem 'register_sources_bods', git: 'https://github.com/openownership/register-sources-bods.git' gem 'register_sources_sk', git: 'https://github.com/openownership/register-sources-sk.git' gem 'register_sources_dk', git: 'https://github.com/openownership/register-sources-dk.git' diff --git a/Gemfile.lock b/Gemfile.lock index 76f7aea6..dc82df23 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GIT GIT remote: https://github.com/openownership/register-sources-bods.git - revision: 9952fbb644b870ecb13899e03bb130762526bf6a + revision: 9fc7ec354ac58480556d0a44d581e9bffd4886a5 specs: register_sources_bods (0.1.0) activesupport (>= 6, < 8) @@ -30,6 +30,19 @@ GIT iso8601 xxhash +GIT + remote: https://github.com/openownership/register-sources-dk.git + revision: b97deb65255d825e11c94108785744936f3abec6 + specs: + register_sources_dk (0.1.0) + activesupport (>= 6, < 8) + countries (~> 4.0.1) + dry-struct (>= 1, < 2) + dry-types (>= 1, < 2) + elasticsearch (>= 7.10, < 8) + iso8601 + xxhash + GIT remote: https://github.com/openownership/register-sources-oc.git revision: 04ef5f2991164717d90f9f0499d082c296335e89 @@ -50,7 +63,7 @@ GIT GIT remote: https://github.com/openownership/register-sources-psc.git - revision: 7dcd200bddca4c99497d0676a9fd5bcacf807939 + revision: c9755e8e37bc095b2e580c2c14f9d65c5f33f5cc specs: register_sources_psc (0.1.0) activesupport (>= 6, < 8) @@ -61,6 +74,19 @@ GIT iso8601 xxhash +GIT + remote: https://github.com/openownership/register-sources-sk.git + revision: b96fbdaa1d9bb7103271b616fa1ba74582b44fe4 + specs: + register_sources_sk (0.1.0) + activesupport (>= 6, < 8) + countries (~> 4.0.1) + dry-struct (>= 1, < 2) + dry-types (>= 1, < 2) + elasticsearch (>= 7.10, < 8) + iso8601 + xxhash + GEM remote: https://rubygems.org/ specs: @@ -566,8 +592,10 @@ DEPENDENCIES redcarpet (~> 3.5) register_common! register_sources_bods! + register_sources_dk! register_sources_oc! register_sources_psc! + register_sources_sk! rexml (~> 3.2) roadie-rails (~> 2.3) rollbar (~> 3.3) diff --git a/app/repositories/datasources.json b/app/repositories/datasources.json index 21daf098..b3e0301d 100644 --- a/app/repositories/datasources.json +++ b/app/repositories/datasources.json @@ -1,97 +1,5 @@ { "datasources": [ - { - "_id": { - "$oid": "5d5fb1beeda3b30004498790" - }, - "_slugs": [ - "eiti-pilot-data-myanmar" - ], - "created_at": "2019-08-23T09:28:30.192Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Myanmar", - "name": "EITI pilot data - Myanmar", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.192Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=950147039" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498799" - }, - "_slugs": [ - "eiti-pilot-data-mali" - ], - "created_at": "2019-08-23T09:28:30.280Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Mali", - "name": "EITI pilot data - Mali", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.280Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=177493552" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b300044987a0" - }, - "_slugs": [ - "eiti-pilot-data-zambia" - ], - "created_at": "2019-08-23T09:28:30.364Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Zambia", - "name": "EITI pilot data - Zambia", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.364Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1198225001" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878d" - }, - "_slugs": [ - "eiti-pilot-data-ghana" - ], - "created_at": "2019-08-23T09:28:30.161Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Ghana", - "name": "EITI pilot data - Ghana", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.161Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1926655631" - }, { "_id": { "$oid": "5d4d2f1d07963b0004c00778" @@ -114,396 +22,6 @@ "updated_at": "2019-11-01T14:21:25.809Z", "url": "https://rpvs.gov.sk/" }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498795" - }, - "_slugs": [ - "eiti-pilot-data-honduras" - ], - "created_at": "2019-08-23T09:28:30.246Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Honduras", - "name": "EITI pilot data - Honduras", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.246Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=793160552" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498797" - }, - "_slugs": [ - "eiti-pilot-data-madagascar" - ], - "created_at": "2019-08-23T09:28:30.263Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Madagascar", - "name": "EITI pilot data - Madagascar", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.263Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1184544716" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879e" - }, - "_slugs": [ - "eiti-pilot-data-togo" - ], - "created_at": "2019-08-23T09:28:30.349Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Togo", - "name": "EITI pilot data - Togo", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.349Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=389809839" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878e" - }, - "_slugs": [ - "eiti-pilot-data-indonesia" - ], - "created_at": "2019-08-23T09:28:30.170Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Indonesia", - "name": "EITI pilot data - Indonesia", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.170Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=538768558" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879b" - }, - "_slugs": [ - "eiti-pilot-data-norway" - ], - "created_at": "2019-08-23T09:28:30.326Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Norway", - "name": "EITI pilot data - Norway", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.326Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1257742514" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498793" - }, - "_slugs": [ - "eiti-pilot-data-burkina-faso" - ], - "created_at": "2019-08-23T09:28:30.221Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Burkina Faso", - "name": "EITI pilot data - Burkina Faso", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.221Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1984808479" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498791" - }, - "_slugs": [ - "eiti-pilot-data-uk" - ], - "created_at": "2019-08-23T09:28:30.202Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - UK", - "name": "EITI pilot data - UK", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.202Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1503351240" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498792" - }, - "_slugs": [ - "eiti-pilot-data-afghanistan" - ], - "created_at": "2019-08-23T09:28:30.212Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Afghanistan", - "name": "EITI pilot data - Afghanistan", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.212Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=100859567" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498798" - }, - "_slugs": [ - "eiti-pilot-data-liberia" - ], - "created_at": "2019-08-23T09:28:30.271Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Liberia", - "name": "EITI pilot data - Liberia", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.271Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=680199964" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879c" - }, - "_slugs": [ - "eiti-pilot-data-seychelles" - ], - "created_at": "2019-08-23T09:28:30.334Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Seychelles", - "name": "EITI pilot data - Seychelles", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.334Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1527384866" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879d" - }, - "_slugs": [ - "eiti-pilot-data-tanzania" - ], - "created_at": "2019-08-23T09:28:30.342Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Tanzania", - "name": "EITI pilot data - Tanzania", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.342Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=859907622" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878c" - }, - "_slugs": [ - "eiti-pilot-data-democratic-republic-of-congo" - ], - "created_at": "2019-08-23T09:28:30.150Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Democratic Republic of Congo", - "name": "EITI pilot data - Democratic Republic of Congo", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.150Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1443594667" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878a" - }, - "_slugs": [ - "openownership-register" - ], - "created_at": "2019-08-23T09:28:30.123Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": null, - "name": "OpenOwnership Register", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "selfDeclaration" - ], - "updated_at": "2019-08-23T09:28:30.123Z", - "url": "https://register.openownership.org" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878f" - }, - "_slugs": [ - "eiti-pilot-data-nigeria" - ], - "created_at": "2019-08-23T09:28:30.179Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Nigeria", - "name": "EITI pilot data - Nigeria", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.179Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1307797548" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498794" - }, - "_slugs": [ - "eiti-pilot-data-cate-divoire" - ], - "created_at": "2019-08-23T09:28:30.235Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Côte d'Ivoire", - "name": "EITI pilot data - Côte d'Ivoire", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.235Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=150471739" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879a" - }, - "_slugs": [ - "eiti-pilot-data-mauritania" - ], - "created_at": "2019-08-23T09:28:30.289Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Mauritania", - "name": "EITI pilot data - Mauritania", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.289Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=909011938" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b300044987a1" - }, - "_slugs": [ - "ua-edr-register" - ], - "created_at": "2019-08-23T09:28:30.372Z", - "current_statistic_types": [ - - ], - "data_availability": "The Unified State Register of Legal Entities, Individual Entrepreneurs and Public\nFormations has data publicly and freely available as bulk download, via a\n[daily zip file](https://data.gov.ua/dataset/1c7f3815-3259-45e0-bdf1-64dca07ddc10)\n(unstructured data).\n\nThe data is licensed under\n[Creative Commons Attribution (cc-by)](https://creativecommons.org/licenses/by/4.0/).\n\nIn order to conduct a record search via a\n[simple website interface](https://usr.minjust.gov.ua/), one has to file a\nrequest and pay a small fee.\n", - "document_id": "Ukraine EDR", - "name": "Ukraine Consolidated State Registry (Edinyy Derzhavnyj Reestr [EDR])", - "overview": "Beneficial ownership data in Ukraine has been publicly available as open data in\nthe Unified State Register of Legal Entities and Individual Entrepreneurs (USR)\nsince 2017. We regularly imported this data until September 2020, when a data\nformat change caused us to pause imports indefinitely.\n\nUkrainian law refers to ‘controllers’ (контролери) as opposed to ‘founders’\n(засновниками) to differentiate beneficial ownership from legal\nownership/shareholding. ‘Founders’ can be legal entities or natural persons.\n\nA beneficial owner of a legal entity is a natural person who directly or\nindirectly:\n\n- Has the right to execute decisive influence on the management or economic\n activity of a legal entity\n- Has decisive influence on the composition and decisions of the entity’s\n managing body\n- Has at least 25% of shares or voting rights of the entity\n\nNominees and agents of such rights cannot be named as beneficial owners.\n\n## Exclusions and exemptions\n\nIf the ‘founders’ of a legal entity are both natural persons and the beneficial\nowners of the entity, then the law relieves them from the duty to separately\ndeclare beneficial ownership (as such information is collected about the founders\nalready).\n\nJoint stock companies and listed companies are not obliged to report their\nbeneficial ownership data in the USR, however they must report owners of 10% and\nabove of company shares to the Stock Market Infrastructure Development Agency\n([SMIDA](https://smida.gov.ua/)).\n\nFinally, the following entities are excluded from the scope of the law: civic\nformations, attorney associations, trade chambers, associations of co-owners of\napartment buildings, religious organisations, state authorities, local\nself-government bodies and their associations, state and municipal\nenterprises/establishments/organisations.\n", - "timeline_url": "https://twitter.com/sheislaurence/timelines/1111242328805650433?ref_src=twsrc%5Etfw", - "types": [ - "officialRegister" - ], - "updated_at": "2021-12-17T17:55:14.831Z", - "url": "https://data.gov.ua/dataset/1c7f3815-3259-45e0-bdf1-64dca07ddc10" - }, { "_id": { "$oid": "5ca49242b51e4f000403da6b" @@ -685,29 +203,6 @@ "updated_at": "2021-12-17T17:55:14.855Z", "url": "http://download.companieshouse.gov.uk/en_pscdata.html" }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449879f" - }, - "_slugs": [ - "eiti-pilot-data-trinidad-and-tobago" - ], - "created_at": "2019-08-23T09:28:30.357Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Trinidad and Tobago", - "name": "EITI pilot data - Trinidad and Tobago", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.357Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1697865173" - }, { "_id": { "$oid": "5d4d2f1d07963b0004c00779" @@ -729,52 +224,6 @@ ], "updated_at": "2019-11-01T14:21:27.022Z", "url": "https://cvr.dk" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b3000449878b" - }, - "_slugs": [ - "eiti-pilot-data-cameroon" - ], - "created_at": "2019-08-23T09:28:30.137Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Cameroon", - "name": "EITI pilot data - Cameroon", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.137Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=1944716262" - }, - { - "_id": { - "$oid": "5d5fb1beeda3b30004498796" - }, - "_slugs": [ - "eiti-pilot-data-kyrgyz-republic" - ], - "created_at": "2019-08-23T09:28:30.255Z", - "current_statistic_types": [ - - ], - "data_availability": null, - "document_id": "EITI Structured Data - Kyrgyz Republic", - "name": "EITI pilot data - Kyrgyz Republic", - "overview": null, - "timeline_url": null, - "types": [ - "thirdParty", - "primaryResearch" - ], - "updated_at": "2019-08-23T09:28:30.255Z", - "url": "https://docs.google.com/spreadsheets/d/1OKl6oe6RbYicPIZEGljYZy29M06Pm4vGvGDoNzq6dV4/export?format=csv&gid=677766885" } ] }