From 6ec1e9ed879b59788c57c033680c5dafc54e8861 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sat, 14 May 2022 12:16:12 +0530 Subject: [PATCH 1/2] Rename match clause to if for better readability --- docs/topics/vortex_by_example.rst | 12 ++++++------ tests/fixtures/mapping-inheritance-invalid.yml | 8 ++++---- tests/fixtures/mapping-inheritance-no-default.yml | 6 +++--- tests/fixtures/mapping-inheritance.yml | 6 +++--- tests/fixtures/mapping-merge-multiple-local.yml | 8 ++++---- tests/fixtures/mapping-merge-multiple-remote.yml | 4 ++-- tests/fixtures/mapping-rule-argument-based.yml | 2 +- tests/fixtures/mapping-rule-execute.yml | 8 ++++---- tests/fixtures/mapping-rule-tool-limits.yml | 4 ++-- tests/fixtures/mapping-rules-changed-extra.yml | 2 +- tests/fixtures/mapping-rules-changed.yml | 12 ++++++------ tests/fixtures/mapping-rules-extra.yml | 2 +- tests/fixtures/mapping-rules.yml | 12 ++++++------ tests/fixtures/scenario-admin-group-user.yml | 2 +- tests/fixtures/scenario-jenkins-bot-user.yml | 2 +- .../fixtures/scenario-job-too-small-for-highmem.yml | 4 ++-- tests/fixtures/scenario-node-offline-high-cpu.yml | 4 ++-- tests/fixtures/scenario-too-many-highmem-jobs.yml | 4 ++-- .../fixtures/scenario-trinity-job-too-much-data.yml | 4 ++-- tests/fixtures/scenario-trinity-job-with-rules.yml | 2 +- tests/fixtures/scenario-usegalaxy-dev.yml | 5 ++--- vortex/core/entities.py | 4 ++-- 22 files changed, 58 insertions(+), 59 deletions(-) diff --git a/docs/topics/vortex_by_example.rst b/docs/topics/vortex_by_example.rst index 2e532ef..fcdc0f0 100644 --- a/docs/topics/vortex_by_example.rst +++ b/docs/topics/vortex_by_example.rst @@ -181,7 +181,7 @@ Rules provide a means by which to conditionally change entity requirements. mem: cores * 3 rules: - id: my_overridable_rule - match: input_size < 5 + if: input_size < 5 fail: We don't run piddling datasets of {input_size}GB bwa: scheduling: @@ -189,22 +189,22 @@ Rules provide a means by which to conditionally change entity requirements. - pulsar rules: - id: my_overridable_rule - match: input_size < 1 + if: input_size < 1 fail: We don't run piddling datasets - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 execute: | from galaxy.jobs.mapper import JobNotReadyException raise JobNotReadyException() - - match: input_size > 10 and input_size < 20 + - if: input_size > 10 and input_size < 20 scheduling: require: - highmem - - match: input_size >= 20 + - if: input_size >= 20 fail: Input size: {input_size} is too large shouldn't run -The match clause can contain arbitrary python code, including multi-line python code. The only requirement is that the +The if clause can contain arbitrary python code, including multi-line python code. The only requirement is that the last statement in the code block must evaluate to a boolean value. In this example, the `input_size` variable is an automatically available contextual variable which is computed by totalling the sizes of all inputs to the job. Additional available variables include app, job, tool, and user. diff --git a/tests/fixtures/mapping-inheritance-invalid.yml b/tests/fixtures/mapping-inheritance-invalid.yml index 4d2104b..a3ab4f3 100644 --- a/tests/fixtures/mapping-inheritance-invalid.yml +++ b/tests/fixtures/mapping-inheritance-invalid.yml @@ -18,7 +18,7 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: gpus: 2 @@ -26,14 +26,14 @@ tools: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 - - match: input_size > 10 and input_size < 20 + - if: input_size > 10 and input_size < 20 scheduling: require: - highmem - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run trinity: inherits: bwa diff --git a/tests/fixtures/mapping-inheritance-no-default.yml b/tests/fixtures/mapping-inheritance-no-default.yml index 0e5a3d7..7fa619c 100644 --- a/tests/fixtures/mapping-inheritance-no-default.yml +++ b/tests/fixtures/mapping-inheritance-no-default.yml @@ -15,7 +15,7 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: gpus: 2 @@ -23,10 +23,10 @@ tools: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run trinity: inherits: bwa diff --git a/tests/fixtures/mapping-inheritance.yml b/tests/fixtures/mapping-inheritance.yml index 23acacf..902f9fd 100644 --- a/tests/fixtures/mapping-inheritance.yml +++ b/tests/fixtures/mapping-inheritance.yml @@ -18,7 +18,7 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: gpus: 2 @@ -26,10 +26,10 @@ tools: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run trinity: inherits: bwa diff --git a/tests/fixtures/mapping-merge-multiple-local.yml b/tests/fixtures/mapping-merge-multiple-local.yml index 618ce9c..525d7de 100644 --- a/tests/fixtures/mapping-merge-multiple-local.yml +++ b/tests/fixtures/mapping-merge-multiple-local.yml @@ -16,22 +16,22 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: scheduling: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 mem: cores * 4 - id: highmem_rule - match: input_size > 40 + if: input_size > 40 fail: "" scheduling: require: - highmem - - match: input_size >= 20 and input_size <= 30 + - if: input_size >= 20 and input_size <= 30 fail: Too much data, shouldn't run users: fairycake@vortex.org: diff --git a/tests/fixtures/mapping-merge-multiple-remote.yml b/tests/fixtures/mapping-merge-multiple-remote.yml index 2ea08dd..3760900 100644 --- a/tests/fixtures/mapping-merge-multiple-remote.yml +++ b/tests/fixtures/mapping-merge-multiple-remote.yml @@ -7,12 +7,12 @@ tools: mem: 8 env: {} rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: rules: - id: highmem_rule - match: input_size > 40 + if: input_size > 40 fail: a different kind of error scheduling: require: diff --git a/tests/fixtures/mapping-rule-argument-based.yml b/tests/fixtures/mapping-rule-argument-based.yml index 6633b2d..7426ca0 100644 --- a/tests/fixtures/mapping-rule-argument-based.yml +++ b/tests/fixtures/mapping-rule-argument-based.yml @@ -14,7 +14,7 @@ tools: - general limbo: rules: - - match: | + - if: | helpers.job_args_match(job, app, {'input_opts': {'db_selector': 'db'}}) scheduling: prefer: diff --git a/tests/fixtures/mapping-rule-execute.yml b/tests/fixtures/mapping-rule-execute.yml index aa4d6de..3d5ee7d 100644 --- a/tests/fixtures/mapping-rule-execute.yml +++ b/tests/fixtures/mapping-rule-execute.yml @@ -17,24 +17,24 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: scheduling: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 execute: | from galaxy.jobs.mapper import JobNotReadyException raise JobNotReadyException() - - match: input_size > 10 and input_size < 20 + - if: input_size > 10 and input_size < 20 scheduling: require: - highmem - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run destinations: diff --git a/tests/fixtures/mapping-rule-tool-limits.yml b/tests/fixtures/mapping-rule-tool-limits.yml index 2867031..0e1e5d3 100644 --- a/tests/fixtures/mapping-rule-tool-limits.yml +++ b/tests/fixtures/mapping-rule-tool-limits.yml @@ -14,14 +14,14 @@ tools: - general toolshed.g2.bx.psu.edu/repos/rnateam/mafft/rbc_mafft/.*: rules: - - match: | + - if: | helpers.concurrent_job_count_for_tool(app, tool, user) >= 2 # concurrent jobs per user execute: | from galaxy.jobs.mapper import JobNotReadyException raise JobNotReadyException() toolshed.g2.bx.psu.edu/repos/artbio/repenrich/repenrich/.*: rules: - - match: | + - if: | helpers.concurrent_job_count_for_tool(app, tool) >= 3 # total concurrent jobs execute: | from galaxy.jobs.mapper import JobNotReadyException diff --git a/tests/fixtures/mapping-rules-changed-extra.yml b/tests/fixtures/mapping-rules-changed-extra.yml index a6ea145..8e431b8 100644 --- a/tests/fixtures/mapping-rules-changed-extra.yml +++ b/tests/fixtures/mapping-rules-changed-extra.yml @@ -5,6 +5,6 @@ tools: - pulsar rules: - id: firstrule - match: input_size <= 10 + if: input_size <= 10 cores: 5 mem: cores * 4 diff --git a/tests/fixtures/mapping-rules-changed.yml b/tests/fixtures/mapping-rules-changed.yml index fd67c44..102f8eb 100644 --- a/tests/fixtures/mapping-rules-changed.yml +++ b/tests/fixtures/mapping-rules-changed.yml @@ -17,21 +17,21 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: scheduling: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 - - match: input_size > 10 and input_size < 20 + - if: input_size > 10 and input_size < 20 scheduling: require: - highmem - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run users: fairycake@vortex.org: @@ -47,10 +47,10 @@ users: require: - earth rules: - - match: True + - if: True cores: 6 mem: cores * 6 - - match: input_size >= 5 + - if: input_size >= 5 fail: Just because improbable@vortex.org: scheduling: diff --git a/tests/fixtures/mapping-rules-extra.yml b/tests/fixtures/mapping-rules-extra.yml index efd1b62..c2bd68d 100644 --- a/tests/fixtures/mapping-rules-extra.yml +++ b/tests/fixtures/mapping-rules-extra.yml @@ -5,6 +5,6 @@ tools: - pulsar rules: - id: firstrule - match: input_size <= 10 + if: input_size <= 10 cores: 3 mem: cores * 4 diff --git a/tests/fixtures/mapping-rules.yml b/tests/fixtures/mapping-rules.yml index 915f3ca..10e1977 100644 --- a/tests/fixtures/mapping-rules.yml +++ b/tests/fixtures/mapping-rules.yml @@ -17,21 +17,21 @@ tools: reject: - pulsar rules: - - match: input_size < 5 + - if: input_size < 5 fail: We don't run piddling datasets bwa: scheduling: require: - pulsar rules: - - match: input_size <= 10 + - if: input_size <= 10 cores: 4 mem: cores * 4 - - match: input_size > 10 and input_size < 20 + - if: input_size > 10 and input_size < 20 scheduling: require: - highmem - - match: input_size >= 20 + - if: input_size >= 20 fail: Too much data, shouldn't run users: fairycake@vortex.org: @@ -47,10 +47,10 @@ users: require: - earth rules: - - match: True + - if: True cores: 6 mem: cores * 6 - - match: input_size >= 5 + - if: input_size >= 5 fail: Just because improbable@vortex.org: scheduling: diff --git a/tests/fixtures/scenario-admin-group-user.yml b/tests/fixtures/scenario-admin-group-user.yml index 758fbd8..2ff4ccf 100644 --- a/tests/fixtures/scenario-admin-group-user.yml +++ b/tests/fixtures/scenario-admin-group-user.yml @@ -46,7 +46,7 @@ tools: reject: - offline rules: - - match: input_size >= 0.1 + - if: input_size >= 0.1 scheduling: require: - highmem diff --git a/tests/fixtures/scenario-jenkins-bot-user.yml b/tests/fixtures/scenario-jenkins-bot-user.yml index ba259fb..b0be2bc 100644 --- a/tests/fixtures/scenario-jenkins-bot-user.yml +++ b/tests/fixtures/scenario-jenkins-bot-user.yml @@ -46,7 +46,7 @@ tools: reject: - offline rules: - - match: input_size >= 0.1 + - if: input_size >= 0.1 scheduling: require: - highmem diff --git a/tests/fixtures/scenario-job-too-small-for-highmem.yml b/tests/fixtures/scenario-job-too-small-for-highmem.yml index 3293747..5194ade 100644 --- a/tests/fixtures/scenario-job-too-small-for-highmem.yml +++ b/tests/fixtures/scenario-job-too-small-for-highmem.yml @@ -46,9 +46,9 @@ tools: reject: - offline rules: - - match: input_size >= 10 + - if: input_size >= 10 cores: 16 - - match: input_size >= 20 + - if: input_size >= 20 scheduling: require: - highmem diff --git a/tests/fixtures/scenario-node-offline-high-cpu.yml b/tests/fixtures/scenario-node-offline-high-cpu.yml index b31d2b9..e1cc585 100644 --- a/tests/fixtures/scenario-node-offline-high-cpu.yml +++ b/tests/fixtures/scenario-node-offline-high-cpu.yml @@ -46,9 +46,9 @@ tools: reject: - offline rules: - - match: input_size >= 10 + - if: input_size >= 10 cores: 16 - - match: input_size >= 20 + - if: input_size >= 20 scheduling: require: - highmem diff --git a/tests/fixtures/scenario-too-many-highmem-jobs.yml b/tests/fixtures/scenario-too-many-highmem-jobs.yml index 1e457b2..d0c8c98 100644 --- a/tests/fixtures/scenario-too-many-highmem-jobs.yml +++ b/tests/fixtures/scenario-too-many-highmem-jobs.yml @@ -47,7 +47,7 @@ tools: reject: - offline rules: - - match: input_size >= 0.1 + - if: input_size >= 0.1 scheduling: require: - highmem @@ -55,7 +55,7 @@ tools: users: default: rules: - - match: | + - if: | from galaxy.jobs.rule_helper import RuleHelper from vortex.core.entities import TagType diff --git a/tests/fixtures/scenario-trinity-job-too-much-data.yml b/tests/fixtures/scenario-trinity-job-too-much-data.yml index 835da90..a81790e 100644 --- a/tests/fixtures/scenario-trinity-job-too-much-data.yml +++ b/tests/fixtures/scenario-trinity-job-too-much-data.yml @@ -46,11 +46,11 @@ tools: reject: - offline rules: - - match: input_size >= 0.1 + - if: input_size >= 0.1 scheduling: require: - highmem - - match: input_size >= 200 + - if: input_size >= 200 fail: "Input file size of {input_size}GB is > maximum allowed 200GB limit" destinations: slurm: diff --git a/tests/fixtures/scenario-trinity-job-with-rules.yml b/tests/fixtures/scenario-trinity-job-with-rules.yml index 67c055a..d500c58 100644 --- a/tests/fixtures/scenario-trinity-job-with-rules.yml +++ b/tests/fixtures/scenario-trinity-job-with-rules.yml @@ -46,7 +46,7 @@ tools: reject: - offline rules: - - match: input_size >= 0.1 + - if: input_size >= 0.1 scheduling: require: - highmem diff --git a/tests/fixtures/scenario-usegalaxy-dev.yml b/tests/fixtures/scenario-usegalaxy-dev.yml index 690b5f5..bdad180 100644 --- a/tests/fixtures/scenario-usegalaxy-dev.yml +++ b/tests/fixtures/scenario-usegalaxy-dev.yml @@ -35,14 +35,13 @@ tools: prefer: - pulsar rules: - - match: input_size >= 0.2 + - if: input_size >= 0.2 cores: 16 users: default: rules: - - match: | - False + - if: False fail: "You cannot have more than 4 high-mem jobs running concurrently" destinations: slurm: diff --git a/vortex/core/entities.py b/vortex/core/entities.py index 7c9b86e..352164c 100644 --- a/vortex/core/entities.py +++ b/vortex/core/entities.py @@ -481,7 +481,7 @@ def from_dict(loader, entity_dict): params=entity_dict.get('params'), tags=entity_dict.get('scheduling'), inherits=entity_dict.get('inherits'), - match=entity_dict.get('match'), + match=entity_dict.get('if'), execute=entity_dict.get('execute'), fail=entity_dict.get('fail') ) @@ -494,7 +494,7 @@ def override(self, entity): return new_entity def __repr__(self): - return super().__repr__() + f", match={self.match[:10] if self.match else ''}, " \ + return super().__repr__() + f", if={self.match[:10] if self.match else ''}, " \ f"execute={self.execute[:10] if self.execute else ''}, " \ f"fail={self.fail[:10] if self.fail else ''}" From 652712c6c65f0b8d57e04d86a1d33133087c6c79 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sat, 14 May 2022 21:16:32 +0530 Subject: [PATCH 2/2] Also support deprecated match clause for now --- vortex/core/entities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vortex/core/entities.py b/vortex/core/entities.py index 352164c..a2617ec 100644 --- a/vortex/core/entities.py +++ b/vortex/core/entities.py @@ -481,7 +481,8 @@ def from_dict(loader, entity_dict): params=entity_dict.get('params'), tags=entity_dict.get('scheduling'), inherits=entity_dict.get('inherits'), - match=entity_dict.get('if'), + # TODO: Remove deprecated match clause in future + match=entity_dict.get('if') or entity_dict.get('match'), execute=entity_dict.get('execute'), fail=entity_dict.get('fail') )