From fd3c9397a80024663a9d156fdb7f2c25f127946a Mon Sep 17 00:00:00 2001 From: cat-bro Date: Fri, 21 Apr 2023 23:34:05 +1000 Subject: [PATCH 1/3] This test passes. What is going wrong? --- tests/fixtures/mapping-destinations.yml | 14 ++++++++++++++ tests/test_mapper_destinations.py | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/fixtures/mapping-destinations.yml b/tests/fixtures/mapping-destinations.yml index 3ff4293..f504a1c 100644 --- a/tests/fixtures/mapping-destinations.yml +++ b/tests/fixtures/mapping-destinations.yml @@ -19,6 +19,7 @@ tools: accept: reject: - pulsar + - offline rules: - if: input_size < 2 fail: Data size too small @@ -100,6 +101,14 @@ tools: require: - test_of_min_mem_accepted +users: + pulsar_canberra_user@act.au: + scheduling: + accept: + - offline + require: + - pulsar-canberra + destinations: local: runner: local @@ -240,3 +249,8 @@ destinations: scheduling: require: - test_of_min_mem_accepted + pulsar-canberra: + runner: k8s + scheduling: + accept: + - pulsar-canberra diff --git a/tests/test_mapper_destinations.py b/tests/test_mapper_destinations.py index f3b0a5f..8ca3889 100644 --- a/tests/test_mapper_destinations.py +++ b/tests/test_mapper_destinations.py @@ -176,3 +176,15 @@ def test_min_accepted_mem_honoured(self): tool = mock_galaxy.Tool('tool_for_testing_min_mem_acceptance_match') destination = self._map_to_destination(tool, user, datasets, tpv_config_paths=[config]) self.assertEqual(destination.id, "destination_with_min_mem_accepted") + + def test_user_map_to_destination_accepting_offline(self): + user = mock_galaxy.User('albo', 'pulsar_canberra_user@act.au') + + config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-destinations.yml') + + tool = mock_galaxy.Tool('some_assembler') + datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=12 * 1024 ** 3))] + destination = self._map_to_destination(tool, user, datasets, tpv_config_paths=[config]) + self.assertEqual(destination.id, "pulsar-canberra") + + \ No newline at end of file From d7af3534f7a93ced1decdc49861177d6e422aa69 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Mon, 24 Apr 2023 22:23:18 +1000 Subject: [PATCH 2/3] make it fail --- tests/fixtures/mapping-destinations.yml | 20 +++++++++++++++----- tests/test_mapper_destinations.py | 4 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/mapping-destinations.yml b/tests/fixtures/mapping-destinations.yml index f504a1c..1f3816f 100644 --- a/tests/fixtures/mapping-destinations.yml +++ b/tests/fixtures/mapping-destinations.yml @@ -27,6 +27,11 @@ tools: scheduling: require: - pulsar + toolshed_hifiasm: + cores: 1000 + rules: + - if: input_size < 1000 + cores: 2 inheritance_test_tool: scheduling: accept: @@ -103,11 +108,15 @@ tools: users: pulsar_canberra_user@act.au: - scheduling: - accept: - - offline - require: - - pulsar-canberra + rules: + - id: pulsar_user_mapping_rule + if: tool.id.startswith('toolshed') or tool.id.startswith('testtoolshed') + scheduling: + accept: + - pulsar # pulsar must be in require/prefer/accept for a job to be scheduled to pulsar + - offline # allow scheduling to a destination even when it is offline + require: + - pulsar-canberra destinations: local: @@ -251,6 +260,7 @@ destinations: - test_of_min_mem_accepted pulsar-canberra: runner: k8s + max_accepted_cores: 32 scheduling: accept: - pulsar-canberra diff --git a/tests/test_mapper_destinations.py b/tests/test_mapper_destinations.py index 8ca3889..72d3f33 100644 --- a/tests/test_mapper_destinations.py +++ b/tests/test_mapper_destinations.py @@ -182,9 +182,9 @@ def test_user_map_to_destination_accepting_offline(self): config = os.path.join(os.path.dirname(__file__), 'fixtures/mapping-destinations.yml') - tool = mock_galaxy.Tool('some_assembler') + tool = mock_galaxy.Tool('toolshed_hifiasm') datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=12 * 1024 ** 3))] destination = self._map_to_destination(tool, user, datasets, tpv_config_paths=[config]) self.assertEqual(destination.id, "pulsar-canberra") - + self.assertEqual(destination.params['yes_cat'], 'really') \ No newline at end of file From 6f3ba54b6e983309c4f0c20f328463a8d57adaa3 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Tue, 25 Apr 2023 19:49:09 +0530 Subject: [PATCH 3/3] Fix incorrect copying of cores/mem/gpus when evaluating multiple rules --- docs/topics/tpv_by_example.rst | 2 +- tests/test_mapper_destinations.py | 2 -- tpv/core/entities.py | 6 +++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/topics/tpv_by_example.rst b/docs/topics/tpv_by_example.rst index 9dcaed2..dbef5f2 100644 --- a/docs/topics/tpv_by_example.rst +++ b/docs/topics/tpv_by_example.rst @@ -34,7 +34,7 @@ at each destination (optional). The tools are matched by tool id, and can be a r resource requirements can also be computed as python expressions. If resource requirements are defined at the destination, TPV will check whether the job will fit. For example, hisat2 will not schedule on `general_pulsar_1` as it has insufficient cores. If resource requirements are omitted in the tool or destination, it is considered a match. -Note that TPV only considers destinations defined in its own config file, and ignore destinations in job_conf.yml. +Note that TPV only considers destinations defined in its own config file, and ignores destinations in job_conf.yml. Default inheritance ------------------- diff --git a/tests/test_mapper_destinations.py b/tests/test_mapper_destinations.py index 72d3f33..0b5823f 100644 --- a/tests/test_mapper_destinations.py +++ b/tests/test_mapper_destinations.py @@ -186,5 +186,3 @@ def test_user_map_to_destination_accepting_offline(self): datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=12 * 1024 ** 3))] destination = self._map_to_destination(tool, user, datasets, tpv_config_paths=[config]) self.assertEqual(destination.id, "pulsar-canberra") - self.assertEqual(destination.params['yes_cat'], 'really') - \ No newline at end of file diff --git a/tpv/core/entities.py b/tpv/core/entities.py index 9738116..266a304 100644 --- a/tpv/core/entities.py +++ b/tpv/core/entities.py @@ -485,9 +485,9 @@ def evaluate(self, context): if rule.is_matching(context): rule = rule.evaluate(context) new_entity = rule.inherit(new_entity) - new_entity.gpus = rule.gpus or self.gpus - new_entity.cores = rule.cores or self.cores - new_entity.mem = rule.mem or self.mem + new_entity.gpus = rule.gpus or new_entity.gpus + new_entity.cores = rule.cores or new_entity.cores + new_entity.mem = rule.mem or new_entity.mem new_entity.id = f"{new_entity.id}, Rule: {rule.id}" context.update({ 'entity': new_entity