From 6a9e7a760db8e9095088958c497f4b9d613c6290 Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 7 Nov 2024 10:40:18 -0600 Subject: [PATCH 01/14] add explicit primary key to source browse --- server/app/models/materialized_views/source_browse_table_row.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app/models/materialized_views/source_browse_table_row.rb b/server/app/models/materialized_views/source_browse_table_row.rb index 2fb22521e..97b5f53bb 100644 --- a/server/app/models/materialized_views/source_browse_table_row.rb +++ b/server/app/models/materialized_views/source_browse_table_row.rb @@ -1,4 +1,5 @@ class MaterializedViews::SourceBrowseTableRow < MaterializedViews::MaterializedView + self.primary_key = :id has_and_belongs_to_many :clinical_trials, join_table: :clinical_trials_sources, foreign_key: :source_id enum source_type: ['PubMed', 'ASCO', 'ASH'] From eccb5bd191fcf29427513535d952f41af2a47052 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 04:22:13 +0000 Subject: [PATCH 02/14] Bump cross-spawn from 7.0.3 to 7.0.6 in /client Bumps [cross-spawn](https://github.com/moxystudio/node-cross-spawn) from 7.0.3 to 7.0.6. - [Changelog](https://github.com/moxystudio/node-cross-spawn/blob/master/CHANGELOG.md) - [Commits](https://github.com/moxystudio/node-cross-spawn/compare/v7.0.3...v7.0.6) --- updated-dependencies: - dependency-name: cross-spawn dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/yarn.lock b/client/yarn.lock index 7cee3bf40..b2ecf72e6 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -5367,9 +5367,9 @@ cross-inspect@1.0.0: tslib "^2.4.0" cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" From f4dbeefcc7a2f2730110789d1b4da02eba8493e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 22:17:44 +0000 Subject: [PATCH 03/14] Bump rails-html-sanitizer from 1.6.0 to 1.6.1 in /server Bumps [rails-html-sanitizer](https://github.com/rails/rails-html-sanitizer) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/rails/rails-html-sanitizer/releases) - [Changelog](https://github.com/rails/rails-html-sanitizer/blob/main/CHANGELOG.md) - [Commits](https://github.com/rails/rails-html-sanitizer/compare/v1.6.0...v1.6.1) --- updated-dependencies: - dependency-name: rails-html-sanitizer dependency-type: indirect ... Signed-off-by: dependabot[bot] --- server/Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 7605e04b6..67870d6dd 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -237,7 +237,7 @@ GEM listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.22.0) + loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -274,17 +274,17 @@ GEM net-protocol net-ssh (7.2.1) nio4r (2.7.3) - nokogiri (1.16.7-aarch64-linux) + nokogiri (1.16.8-aarch64-linux) racc (~> 1.4) - nokogiri (1.16.7-arm-linux) + nokogiri (1.16.8-arm-linux) racc (~> 1.4) - nokogiri (1.16.7-arm64-darwin) + nokogiri (1.16.8-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86-linux) + nokogiri (1.16.8-x86-linux) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.16.8-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.16.8-x86_64-linux) racc (~> 1.4) oauth2 (1.4.11) faraday (>= 0.17.3, < 3.0) @@ -369,9 +369,9 @@ GEM activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.1) loofah (~> 2.21) - nokogiri (~> 1.14) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) railties (7.1.4.1) actionpack (= 7.1.4.1) activesupport (= 7.1.4.1) From 46262bfb639223fe737b91c3c3a01f8842bccd39 Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Wed, 4 Dec 2024 11:29:59 -0600 Subject: [PATCH 04/14] add default placeholder text to country select on profile page --- .../app/forms/components/user-profile/user-profile.form.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/app/forms/components/user-profile/user-profile.form.html b/client/src/app/forms/components/user-profile/user-profile.form.html index 0d1419291..a73d84e95 100644 --- a/client/src/app/forms/components/user-profile/user-profile.form.html +++ b/client/src/app/forms/components/user-profile/user-profile.form.html @@ -46,6 +46,8 @@ Date: Wed, 4 Dec 2024 12:35:11 -0600 Subject: [PATCH 05/14] use actual csv library to ensure correct quoting, squish internal whitespace --- server/app/jobs/generate_tsvs.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/app/jobs/generate_tsvs.rb b/server/app/jobs/generate_tsvs.rb index 5cf15d4a9..4970fc41d 100644 --- a/server/app/jobs/generate_tsvs.rb +++ b/server/app/jobs/generate_tsvs.rb @@ -1,12 +1,14 @@ +require 'csv' class GenerateTsvs < ApplicationJob def perform ensure_downloads_directory_exists tsvs_to_generate.each do |e| begin tmp_file = tmp_file(e.file_name) - tmp_file.puts(e.headers.join("\t")) + tmp_file.puts CSV.generate_line(e.headers, col_sep: "\t") e.objects.find_each do |object| - tmp_file.puts(e.row_from_object(object).join("\t")) + row = e.row_from_object(object).map {|col| col.is_a?(String) ? col.squish : col } + tmp_file.puts CSV.generate_line(row, col_sep: "\t", quote_empty: false) end tmp_file.close public_path = public_file_path(e.file_name) From d1dcb3b33be842c28d60af717b9db216f39394ee Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Wed, 4 Dec 2024 12:48:09 -0600 Subject: [PATCH 06/14] check for disease usage in revisions in the case where there is no DOID --- .../lib/importer/disease_ontology_mirror.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/server/app/lib/importer/disease_ontology_mirror.rb b/server/app/lib/importer/disease_ontology_mirror.rb index a72bf3518..2e2639bed 100644 --- a/server/app/lib/importer/disease_ontology_mirror.rb +++ b/server/app/lib/importer/disease_ontology_mirror.rb @@ -88,8 +88,7 @@ def delete_unprocessed_diseases unprocessed_doids.each do |doid| d = Disease.find_by(doid: doid) - revisions = Revision.where(field_name: 'disease_id').where(current_value: d.id).or(Revision.where(field_name: 'disease_id').where(suggested_value: d.id)) - if d.evidence_items.count == 0 && d.assertions.count == 0 && d.source_suggestions.count == 0 && revisions.count == 0 + if is_disease_with_no_relations?(d) d.disease_aliases.clear d.delete else @@ -121,7 +120,7 @@ def delete_unprocessed_diseases end end Disease.where(doid: nil).each do |d| - if d.evidence_items.count == 0 && d.assertions.count == 0 && d.source_suggestions.count == 0 + if is_disease_with_no_relations?(d) d.disease_aliases.clear d.delete elsif ['Solid Tumor', 'Ventricular Dysfunction', 'Acute Mountain Sickness', 'Glioma', 'Low Bone Mineral Density'].include? d.name @@ -135,6 +134,20 @@ def delete_unprocessed_diseases end end + def is_disease_with_no_relations?(d) + d.evidence_items.count == 0 && + d.assertions.count == 0 && + d.source_suggestions.count == 0 && + revisions_count(d) == 0 + end + + def revisions_count(disease) + Revision.where(field_name: 'disease_id') + .where(current_value: disease.id) + .or(Revision.where(field_name: 'disease_id').where(suggested_value: disease.id)) + .count + end + def url_from_doid(doid) URI.parse("https://disease-ontology.org/api/metadata/DOID:#{doid}/") end From 421ad3981f5f99a78b163ec5b894dd8d8b7fc6d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:17:23 +0000 Subject: [PATCH 07/14] Bump cross-spawn from 7.0.3 to 7.0.6 in /server Bumps [cross-spawn](https://github.com/moxystudio/node-cross-spawn) from 7.0.3 to 7.0.6. - [Changelog](https://github.com/moxystudio/node-cross-spawn/blob/master/CHANGELOG.md) - [Commits](https://github.com/moxystudio/node-cross-spawn/compare/v7.0.3...v7.0.6) --- updated-dependencies: - dependency-name: cross-spawn dependency-type: indirect ... Signed-off-by: dependabot[bot] --- server/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/yarn.lock b/server/yarn.lock index dbcbca99d..6320060e9 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1891,9 +1891,9 @@ cosmiconfig@^7.0.0: yaml "^1.10.0" cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" From ef83bec95622365cea32caf4be5ffb289cfc336e Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 10:21:37 -0600 Subject: [PATCH 08/14] add alias matching to MP name filtering --- server/app/graphql/resolvers/top_level_molecular_profiles.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/app/graphql/resolvers/top_level_molecular_profiles.rb b/server/app/graphql/resolvers/top_level_molecular_profiles.rb index 6fbab8f30..189db60b9 100644 --- a/server/app/graphql/resolvers/top_level_molecular_profiles.rb +++ b/server/app/graphql/resolvers/top_level_molecular_profiles.rb @@ -35,7 +35,9 @@ class Resolvers::TopLevelMolecularProfiles < GraphQL::Schema::Resolver ) ids = results.hits.map { |x| x["_id"] } - scope.where(molecular_profiles: { id: ids }) + scope.left_joins(:molecular_profile_aliases).where(molecular_profiles: { id: ids }) + .or(scope.where('molecular_profile_aliases.name ILIKE :query', {query: "%#{value}%"})) + .distinct end option(:gene_id, type: Int, description: "Filter molecular profiles to the CIViC id of the gene(s) involved.") do |scope, value| From f2204915283c332851abce3054ca29e1a83c6c0a Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 10:37:26 -0600 Subject: [PATCH 09/14] add script to ensure example scripts keep working --- .github/workflows/test_example_scripts.yml | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/test_example_scripts.yml diff --git a/.github/workflows/test_example_scripts.yml b/.github/workflows/test_example_scripts.yml new file mode 100644 index 000000000..fc59345c3 --- /dev/null +++ b/.github/workflows/test_example_scripts.yml @@ -0,0 +1,49 @@ +name: Test Example Scripts + +on: + push: + branches: [ main, staging ] + pull_request: + branches: [ main, staging] + +jobs: + run-scripts: + runs-on: 'ubuntu-latest' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + + - name: Install Ruby dependencies + run: gem install httparty + + - name: Run Ruby script + run: ruby examples/fetch_variant_links.rb + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.13' + + - name: Install Python dependencies + run: pip install requests + + - name: Run Python script + run: python examples/fetch_all_variants.py + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: '4.4' + + - name: Install R dependencies + run: | + Rscript -e 'install.packages(c("ghql", "jsonlite", "dplyr"), repos="https://cloud.r-project.org")' + + - name: Run R script + run: Rscript examples/fetch_acmg_code.R From 8c8ce3b34cb013b6da366327702e80a55f1461f8 Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 10:41:53 -0600 Subject: [PATCH 10/14] add missing require statement --- examples/fetch_variant_links.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/fetch_variant_links.rb b/examples/fetch_variant_links.rb index 2b10da217..92f5a1dde 100644 --- a/examples/fetch_variant_links.rb +++ b/examples/fetch_variant_links.rb @@ -1,4 +1,5 @@ require 'httparty' +require 'json' def call_graphql_api(url:, query:, vars:) headers = { From f11b93c611fc01ba62244c89b1c71d3bef5201c1 Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 10:50:30 -0600 Subject: [PATCH 11/14] update variants query in python example --- examples/fetch_all_variants.py | 42 +++++++++++++++------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/examples/fetch_all_variants.py b/examples/fetch_all_variants.py index 2aaa93341..dad648b6c 100644 --- a/examples/fetch_all_variants.py +++ b/examples/fetch_all_variants.py @@ -23,34 +23,28 @@ def run_query(query, variables): id name variantAliases - variantBases - referenceBases - referenceBuild - clinvarIds - alleleRegistryId - ensemblVersion - gene { - name + variantTypes { id - entrezId + soid + description } - hgvsDescriptions - primaryCoordinates { - start - stop - chromosome - representativeTranscript + ... on GeneVariant { + coordinates { + chromosome + start + stop + referenceBases + variantBases + representativeTranscript + referenceBuild + } } - secondaryCoordinates{ - start - stop - chromosome - representativeTranscript + ... on FactorVariant { + name + ncitId } - variantTypes { - id - soid - description + ... on FusionVariant { + viccCompliantName } } } From 1d61c35fbe46475c88f9dc8c0048565109add0cd Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 10:58:06 -0600 Subject: [PATCH 12/14] install r deps inline --- .github/workflows/test_example_scripts.yml | 4 ---- examples/fetch_acmg_code.R | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test_example_scripts.yml b/.github/workflows/test_example_scripts.yml index fc59345c3..4fb80ffec 100644 --- a/.github/workflows/test_example_scripts.yml +++ b/.github/workflows/test_example_scripts.yml @@ -41,9 +41,5 @@ jobs: with: r-version: '4.4' - - name: Install R dependencies - run: | - Rscript -e 'install.packages(c("ghql", "jsonlite", "dplyr"), repos="https://cloud.r-project.org")' - - name: Run R script run: Rscript examples/fetch_acmg_code.R diff --git a/examples/fetch_acmg_code.R b/examples/fetch_acmg_code.R index 0a6db530f..f1ba8346c 100644 --- a/examples/fetch_acmg_code.R +++ b/examples/fetch_acmg_code.R @@ -1,3 +1,4 @@ +install.packages(c("ghql", "jsonlite", "dplyr"), repos="https://cloud.r-project.org") library(ghql) library(jsonlite) library(dplyr) From ef909da46664daab24f5b972bf5e471cbefeb49d Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 11:01:13 -0600 Subject: [PATCH 13/14] run each example in parallel --- .github/workflows/test_example_scripts.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_example_scripts.yml b/.github/workflows/test_example_scripts.yml index 4fb80ffec..1546222f2 100644 --- a/.github/workflows/test_example_scripts.yml +++ b/.github/workflows/test_example_scripts.yml @@ -7,9 +7,8 @@ on: branches: [ main, staging] jobs: - run-scripts: + run-ruby-script: runs-on: 'ubuntu-latest' - steps: - name: Checkout repository uses: actions/checkout@v2 @@ -25,6 +24,12 @@ jobs: - name: Run Ruby script run: ruby examples/fetch_variant_links.rb + run-python-script: + runs-on: 'ubuntu-latest' + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Python uses: actions/setup-python@v2 with: @@ -36,6 +41,12 @@ jobs: - name: Run Python script run: python examples/fetch_all_variants.py + run-r-script: + runs-on: 'ubuntu-latest' + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up R uses: r-lib/actions/setup-r@v2 with: From 4b10b06477c41f24a40631fb21009f8ef2fe0309 Mon Sep 17 00:00:00 2001 From: Adam Coffman Date: Thu, 5 Dec 2024 11:11:27 -0600 Subject: [PATCH 14/14] add missing R package deps --- .github/workflows/test_example_scripts.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_example_scripts.yml b/.github/workflows/test_example_scripts.yml index 1546222f2..f551ea021 100644 --- a/.github/workflows/test_example_scripts.yml +++ b/.github/workflows/test_example_scripts.yml @@ -7,8 +7,9 @@ on: branches: [ main, staging] jobs: - run-ruby-script: + run-scripts: runs-on: 'ubuntu-latest' + steps: - name: Checkout repository uses: actions/checkout@v2 @@ -24,12 +25,6 @@ jobs: - name: Run Ruby script run: ruby examples/fetch_variant_links.rb - run-python-script: - runs-on: 'ubuntu-latest' - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Python uses: actions/setup-python@v2 with: @@ -41,16 +36,15 @@ jobs: - name: Run Python script run: python examples/fetch_all_variants.py - run-r-script: - runs-on: 'ubuntu-latest' - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up R uses: r-lib/actions/setup-r@v2 with: r-version: '4.4' + - name: Install system dependencies for R packages + run: | + sudo apt-get update + sudo apt-get install -y libcurl4-openssl-dev libssl-dev libxml2-dev + - name: Run R script run: Rscript examples/fetch_acmg_code.R