From 0098659cfeec20fdddbce0e7f3ee8a1007381326 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 15:27:02 +0100 Subject: [PATCH 01/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03a1b21ecc48..1d6e882de47a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,12 +11,14 @@ on: - '.github/workflows/**' - 'bin/Powheg/**' - '!**/*.input' + - 'bin/utils/**' pull_request: branches: [ master ] paths: - '.github/workflows/**' - 'bin/Powheg/**' - '!**/*.input' + - 'bin/utils/**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -44,7 +46,14 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + powheg1: + - 'bin/Powheg/**' + powheg2: + - '!**/*.input' - name: Build gridpack uses: nick-fields/retry@v2 with: @@ -54,6 +63,7 @@ jobs: command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat + if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.powheg2 == 'true' run: git diff --exit-code bin/Powheg/test/pwg-stat.dat.${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} - name: Create Pull Request @@ -66,4 +76,25 @@ jobs: title: Automated addition ${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} body: | Automated addition by [create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub action. - Please review the cross section file carefully! \ No newline at end of file + Please review the cross section file carefully! + ############################################################# + req_check_script: + # The type of runner that the job will run on + runs-on: [self-hosted, linux, x64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + bin: + - 'bin/utils/**' + - name: Run the request checking script on local mcm requests + shell: python + if: steps.changes.outputs.bin == 'true' + run: | + python3 request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 + python3 request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00118 \ No newline at end of file From 5faf5d7a99ec9a193153695dd8198a58698e5d63 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 16:39:01 +0100 Subject: [PATCH 02/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1d6e882de47a..564a3f184406 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,8 +52,6 @@ jobs: filters: | powheg1: - 'bin/Powheg/**' - powheg2: - - '!**/*.input' - name: Build gridpack uses: nick-fields/retry@v2 with: @@ -63,7 +61,7 @@ jobs: command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat - if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.powheg2 == 'true' + if: steps.changes.outputs.powheg1 == 'true' run: git diff --exit-code bin/Powheg/test/pwg-stat.dat.${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} - name: Create Pull Request @@ -96,5 +94,17 @@ jobs: shell: python if: steps.changes.outputs.bin == 'true' run: | - python3 request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 - python3 request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00118 \ No newline at end of file + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00118 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00118 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL17wmLHEGEN-00119 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SMP-RunIISummer20UL16GEN-00050 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid HIG-RunIISummer20UL16wmLHEGEN-14374 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TRK-Run3Summer23wmLHEGS-00002 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00007 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00045 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid EXO-Run3Summer22wmLHEGS-01444 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00754 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 From c29a42c5ab69d44763ee58d7d2eee05869afbf03 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:00:05 +0100 Subject: [PATCH 03/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 564a3f184406..853c86be7dcc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,7 @@ jobs: max_attempts: 5 timeout_minutes: 60 retry_on_exit_code: 111 # failed CMSSW setup, likely due to cvmfs connectivity + if: steps.changes.outputs.powheg1 == 'true' command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat From 427e3c2c069b1a7b0c1b896fe5ce259f6d506dfe Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:12:50 +0100 Subject: [PATCH 04/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 853c86be7dcc..7085f4ecbe11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,12 +53,12 @@ jobs: powheg1: - 'bin/Powheg/**' - name: Build gridpack + if: steps.changes.outputs.powheg1 == 'true' uses: nick-fields/retry@v2 with: max_attempts: 5 timeout_minutes: 60 retry_on_exit_code: 111 # failed CMSSW setup, likely due to cvmfs connectivity - if: steps.changes.outputs.powheg1 == 'true' command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat From 8ba68c3a21f77c9a280197cf3b768993bbce2095 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:22:53 +0100 Subject: [PATCH 05/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7085f4ecbe11..f283f96b98ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,8 +52,10 @@ jobs: filters: | powheg1: - 'bin/Powheg/**' + workflow: + - '.github/workflows/**' - name: Build gridpack - if: steps.changes.outputs.powheg1 == 'true' + if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow 'true' uses: nick-fields/retry@v2 with: max_attempts: 5 @@ -62,7 +64,7 @@ jobs: command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat - if: steps.changes.outputs.powheg1 == 'true' + if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow 'true' run: git diff --exit-code bin/Powheg/test/pwg-stat.dat.${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} - name: Create Pull Request @@ -91,9 +93,11 @@ jobs: filters: | bin: - 'bin/utils/**' + workflow: + - '.github/workflows/**' - name: Run the request checking script on local mcm requests shell: python - if: steps.changes.outputs.bin == 'true' + if: steps.changes.outputs.bin == 'true' || steps.changes.outputs.workflow == 'true' run: | python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00118 From cb582b45c2f9c59a5030311b916b869244b0fcee Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:24:22 +0100 Subject: [PATCH 06/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f283f96b98ee..1b95d93ab9ab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,7 +55,7 @@ jobs: workflow: - '.github/workflows/**' - name: Build gridpack - if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow 'true' + if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow == 'true' uses: nick-fields/retry@v2 with: max_attempts: 5 @@ -64,7 +64,7 @@ jobs: command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat - if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow 'true' + if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow == 'true' run: git diff --exit-code bin/Powheg/test/pwg-stat.dat.${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} - name: Create Pull Request From e345bf93d4764da90f64ca50b4e1ad75745e09d5 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:33:55 +0100 Subject: [PATCH 07/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1b95d93ab9ab..eee043881077 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,14 +6,14 @@ name: CI on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ master ] +# branches: [ master ] paths: - '.github/workflows/**' - 'bin/Powheg/**' - '!**/*.input' - 'bin/utils/**' pull_request: - branches: [ master ] +# branches: [ master ] paths: - '.github/workflows/**' - 'bin/Powheg/**' From 6aab1aef9f1374c4f1cc67a5527de9fbc3b65159 Mon Sep 17 00:00:00 2001 From: Efe Date: Thu, 11 Jan 2024 17:42:34 +0100 Subject: [PATCH 08/19] initial attempt workflow for reqcheckscript --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eee043881077..1e02f3c2f90b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,14 +6,14 @@ name: CI on: # Triggers the workflow on push or pull request events but only for the master branch push: -# branches: [ master ] + branches: [ master ] paths: - '.github/workflows/**' - 'bin/Powheg/**' - '!**/*.input' - 'bin/utils/**' pull_request: -# branches: [ master ] + branches: [ master ] paths: - '.github/workflows/**' - 'bin/Powheg/**' @@ -88,7 +88,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 - id: filter + id: changes with: filters: | bin: From 1c198924475add26e1bd2b4a60b7c695703ec776 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 10:16:45 +0100 Subject: [PATCH 09/19] wf for request checking script --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e02f3c2f90b..e37db22bff0a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,8 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 - id: filter + #filter + id: changes with: filters: | powheg1: @@ -88,6 +89,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 + #filter id: changes with: filters: | From 08dc687e1b799156216232f35d216ae3b1ba8b03 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 10:29:17 +0100 Subject: [PATCH 10/19] workflow for request checking scr --- .github/workflows/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e37db22bff0a..a90a8294ef88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,8 +47,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 - #filter - id: changes + id: filter with: filters: | powheg1: @@ -56,7 +55,7 @@ jobs: workflow: - '.github/workflows/**' - name: Build gridpack - if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow == 'true' + if: steps.filter.outputs.powheg1 == 'true' || steps.filter.outputs.workflow == 'true' uses: nick-fields/retry@v2 with: max_attempts: 5 @@ -89,8 +88,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 - #filter - id: changes + id: filter with: filters: | bin: @@ -99,7 +97,7 @@ jobs: - '.github/workflows/**' - name: Run the request checking script on local mcm requests shell: python - if: steps.changes.outputs.bin == 'true' || steps.changes.outputs.workflow == 'true' + if: steps.filter.outputs.bin == 'true' || steps.changes.outputs.workflow == 'true' run: | python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00118 From aea6cfd9b8079dc4f16ec92e528561516ccac080 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 13:17:24 +0100 Subject: [PATCH 11/19] Req check script wf --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a90a8294ef88..5ce9f179fc15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,7 +64,7 @@ jobs: command: docker run --rm -v /cvmfs:/cvmfs -v $PWD:$PWD -w $PWD gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cc7-cms bash bin/Powheg/test/test_build.sh -r ${{ matrix.cmssw_release }} -a ${{ matrix.scram_arch }} -p ${{ matrix.proc }} - name: Check pwg-stat.dat - if: steps.changes.outputs.powheg1 == 'true' || steps.changes.outputs.workflow == 'true' + if: steps.filter.outputs.powheg1 == 'true' || steps.filter.outputs.workflow == 'true' run: git diff --exit-code bin/Powheg/test/pwg-stat.dat.${{ matrix.proc }}_${{ matrix.scram_arch }}_${{ matrix.cmssw_release }} - name: Create Pull Request @@ -97,7 +97,7 @@ jobs: - '.github/workflows/**' - name: Run the request checking script on local mcm requests shell: python - if: steps.filter.outputs.bin == 'true' || steps.changes.outputs.workflow == 'true' + if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00118 From bc7d446236f340486db9697046619d168060ce5f Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 14:24:30 +0100 Subject: [PATCH 12/19] Req check script wf --- .github/workflows/main.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ce9f179fc15..1fe6d6c0e4a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,20 +96,11 @@ jobs: workflow: - '.github/workflows/**' - name: Run the request checking script on local mcm requests - shell: python + #shell: python + uses: actions/setup-python@v4 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.x' if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00226 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00118 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid GEN-Run3Summer22EEwmLHEGS-00403 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00118 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TOP-RunIISummer20UL17wmLHEGEN-00119 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SMP-RunIISummer20UL16GEN-00050 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid B2G-RunIISummer20UL18wmLHEGEN-03308 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid HIG-RunIISummer20UL16wmLHEGEN-14374 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TRK-Run3Summer23wmLHEGS-00002 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00007 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid TSG-Run3Summer23BPixGS-00045 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid EXO-Run3Summer22wmLHEGS-01444 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00754 - python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 + python bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 From 33aa2053d9d0e2191c143627c8bd0ab319ce87cc Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 14:27:40 +0100 Subject: [PATCH 13/19] Req check script wf --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1fe6d6c0e4a1..55f826187018 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,6 +88,10 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 + - uses: actions/setup-python@v4 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.x' id: filter with: filters: | @@ -97,10 +101,6 @@ jobs: - '.github/workflows/**' - name: Run the request checking script on local mcm requests #shell: python - uses: actions/setup-python@v4 - with: - # Semantic version range syntax or exact version of a Python version - python-version: '3.x' if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | python bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 From 775e08a6dcbc3e630ad7a7b3c229749f8169cc5b Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 14:32:52 +0100 Subject: [PATCH 14/19] Req check script wf --- .github/workflows/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55f826187018..5bd44d2115fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,10 +88,6 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 - - uses: actions/setup-python@v4 - with: - # Semantic version range syntax or exact version of a Python version - python-version: '3.x' id: filter with: filters: | @@ -99,6 +95,9 @@ jobs: - 'bin/utils/**' workflow: - '.github/workflows/**' + - uses: actions/setup-python@v4 + with: + python-version: '3.x' - name: Run the request checking script on local mcm requests #shell: python if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' From 43ac4d9c806d0883688f8208134556730a00e586 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 14:42:07 +0100 Subject: [PATCH 15/19] Req check script wf --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5bd44d2115fe..786161e64575 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -97,7 +97,7 @@ jobs: - '.github/workflows/**' - uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.9.18' - name: Run the request checking script on local mcm requests #shell: python if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' From 1d33848c734cfee09f8eb6d34492bbf26cb31cfb Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 14:54:52 +0100 Subject: [PATCH 16/19] Req check script wf --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 786161e64575..c0b7b6ecfd6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,8 @@ jobs: strategy: matrix: proc: [hvq] - cmssw_release: [CMSSW_10_6_30, CMSSW_11_0_1, CMSSW_12_3_1] + #cmssw_release: [CMSSW_10_6_30, CMSSW_11_0_1, CMSSW_12_3_1] + cmssw_release: [CMSSW_10_6_30] include: - cmssw_release: CMSSW_10_6_30 scram_arch: slc7_amd64_gcc700 @@ -97,7 +98,7 @@ jobs: - '.github/workflows/**' - uses: actions/setup-python@v4 with: - python-version: '3.9.18' + python-version: '3.9' - name: Run the request checking script on local mcm requests #shell: python if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' From 5e020dd429b139d2ced724d89abe4ae7422a545b Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 15:01:54 +0100 Subject: [PATCH 17/19] Req check script wf --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0b7b6ecfd6e..d2ecf7aed9c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,11 +96,8 @@ jobs: - 'bin/utils/**' workflow: - '.github/workflows/**' - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - name: Run the request checking script on local mcm requests #shell: python if: steps.filter.outputs.bin == 'true' || steps.filter.outputs.workflow == 'true' run: | - python bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 + python3 bin/utils/request_fragment_check.py --bypass_status --develop --local --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 From fb134c1f35edfe9fb711cc4da429b24f2c3d5380 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 16:01:38 +0100 Subject: [PATCH 18/19] studpid changes just for the github workflow to run without accessing mcm --- ...S-RunIISummer20UL16wmLHEGEN-00768_get_test | 131 ++++++++++++++++++ bin/utils/request_fragment_check.py | 33 +++-- 2 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test diff --git a/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test b/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test new file mode 100644 index 000000000000..de29b34a9454 --- /dev/null +++ b/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test @@ -0,0 +1,131 @@ +#!/bin/bash + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py +./request_fragment_check.py --bypass_status --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/SUS-RunIISummer20UL16wmLHEGEN-00768 --retry 3 --create-dirs -o Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py +[ -s Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh +#!/bin/bash + +export SCRAM_ARCH=slc7_amd64_gcc700 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_10_6_32_patch1/src ] ; then + echo release CMSSW_10_6_32_patch1 already exists +else + scram p CMSSW CMSSW_10_6_32_patch1 +fi +cd CMSSW_10_6_32_patch1/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 28800s +# Margin for validation duration: 30% +# Validation duration with margin: 28800 * (1 - 0.30) = 20160s +# Time per event for each sequence: 30.0000s +# Threads for each sequence: 1 +# Time per event for single thread for each sequence: 1 * 30.0000s = 30.0000s +# Which adds up to 30.0000s per event +# Single core events that fit in validation duration: 20160s / 30.0000s = 672 +# Produced events limit in McM is 10000 +# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 +# Take the minimum of 672 and 10000, but more than 0 -> 672 +# It is estimated that this validation will produce: 672 * 1.0000 = 672 events +EVENTS=672 + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py --python_filename SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:SUS-RunIISummer20UL16wmLHEGEN-00768.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py || exit $? ; + +# Parse values from SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of SUS-RunIISummer20UL16wmLHEGEN-00768 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file +EndOfTestFile + +# Make file executable +chmod +x SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then + CONTAINER_NAME="el7:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then + CONTAINER_NAME="el7:x86_64" +else + echo "Could not find amd64 or x86_64 for el7" + exit 1 +fi +# Run in singularity container +# Mount afs, eos, cvmfs +# Mount /etc/grid-security for xrootd +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run -B /afs -B /eos -B /cvmfs -B /etc/grid-security -B /etc/pki/ca-trust --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 4c746398e7e5..2ef1bab17774 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -11,8 +11,8 @@ import json import ast from datetime import datetime -sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') -from rest import McM +#sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') +#from rest import McM from json import dumps parser = argparse.ArgumentParser( @@ -54,8 +54,11 @@ sys.exit() # Use no-id as identification mode in order not to use a SSO cookie -mcm = McM(id=None, dev=args.dev, debug=args.debug) -mcm_link = mcm.server +if args.local is False: + sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') + from rest import McM + mcm = McM(id=None, dev=args.dev, debug=args.debug) + mcm_link = mcm.server def get_request(prepid): if args.local is False: @@ -473,15 +476,16 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): return requests -if args.ticket is not None: - ticket = args.ticket - ticket = ticket[0] - print("------------------------------------") - print("--> Ticket = "+ticket) - print("------------------------------------") - prepid = [] - for rr in root_requests_from_ticket(ticket): - if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr) +if args.local is False: + if args.ticket is not None: + ticket = args.ticket + ticket = ticket[0] + print("------------------------------------") + print("--> Ticket = "+ticket) + print("------------------------------------") + prepid = [] + for rr in root_requests_from_ticket(ticket): + if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr) @@ -693,7 +697,8 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): os.system('mkdir -p '+my_path+'/eos/'+pi) os.system('mv '+pi+'_tmp '+pi) os.system('cp '+pi+' '+my_path+'/'+pi+'/.') - os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test') + if args.local is False: + os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test') gettest = os.popen('grep cff '+pi+'_get_test'+' | grep curl').read() if os.path.getsize(pi+'_get_test') == 0: print("public/restapi/requests/get_test/ is not acessible for this request. Exiting! Please send an email to 'cms-ppd-pdmv-dev@cern.ch'") From 7376fe9ecb50150d34a0b05877d5bc17c5d7935a Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 16:07:50 +0100 Subject: [PATCH 19/19] changes for wf to run with no access to mcm --- bin/utils/request_fragment_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 2ef1bab17774..1cfb741a98a5 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -11,6 +11,7 @@ import json import ast from datetime import datetime +# #sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') #from rest import McM from json import dumps