From c30b7521b223c3d9701e15e2148d5ed93541b98a Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Tue, 25 Jun 2024 16:58:53 -0700 Subject: [PATCH] Use pathogen-repo-build in GH Action workflows Updates all GH Action workflows for running `nextstrain build` to use the shared `pathogen-repo-build` workflow. Resolves https://github.com/nextstrain/ncov/issues/1113 --- .github/workflows/rebuild-100k.yml | 67 +++++++----------------- .github/workflows/rebuild-country.yml | 56 ++++---------------- .github/workflows/rebuild-gisaid-21L.yml | 54 ++++--------------- .github/workflows/rebuild-gisaid.yml | 54 ++++--------------- .github/workflows/rebuild-open.yml | 54 ++++--------------- 5 files changed, 59 insertions(+), 226 deletions(-) diff --git a/.github/workflows/rebuild-100k.yml b/.github/workflows/rebuild-100k.yml index 0eabc0f01..7d98e1d8b 100644 --- a/.github/workflows/rebuild-100k.yml +++ b/.github/workflows/rebuild-100k.yml @@ -9,13 +9,12 @@ on: jobs: gisaid: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - - - name: Launch GISAID build + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch run: | set -x @@ -23,7 +22,6 @@ jobs: config+=(slack_token=$SLACK_TOKEN) nextstrain build \ - --aws-batch \ --detach \ --cpus 16 \ --memory 31GiB \ @@ -31,14 +29,16 @@ jobs: upload \ --configfile nextstrain_profiles/100k/config-gisaid.yaml \ --config "${config[@]}" \ - --set-threads tree=8 \ - |& tee build-launch-gisaid.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - name: Launch open build + --set-threads tree=8 + artifact-name: gisaid-build-output + + open: + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch run: | set -x @@ -46,7 +46,6 @@ jobs: config+=(slack_token=$SLACK_TOKEN) nextstrain build \ - --aws-batch \ --detach \ --cpus 16 \ --memory 31GiB \ @@ -54,35 +53,5 @@ jobs: upload \ --configfile nextstrain_profiles/100k/config-open.yaml \ --config "${config[@]}" \ - --set-threads tree=8 \ - |& tee build-launch-open.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - - name: Build info - run: | - echo "--> 100k samples for GISAID + Open data rebuilding (using separate AWS jobs)" - echo - echo "--> When completed, the following files will be updated:" - echo "s3://nextstrain-data/files/ncov/open/100k/metadata.tsv.xz" - echo "s3://nextstrain-data/files/ncov/open/100k/sequences.fasta.xz" - echo "s3://nextstrain-ncov-private/100k/metadata.tsv.xz" - echo "s3://nextstrain-ncov-private/100k/sequences.fasta.xz" - echo - echo "--> You can attach to the GISAID AWS job via:" - tail -n1 build-launch-gisaid.log - echo - echo "--> You can attach to the Open AWS job via:" - tail -n1 build-launch-open.log - echo - JOBID=$( tail -n1 build-launch-gisaid.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View the GISAID job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo - JOBID=$( tail -n1 build-launch-open.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View the Open job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo + --set-threads tree=8 + artifact-name: open-build-output diff --git a/.github/workflows/rebuild-country.yml b/.github/workflows/rebuild-country.yml index dd6c4a60f..e99f84aa7 100644 --- a/.github/workflows/rebuild-country.yml +++ b/.github/workflows/rebuild-country.yml @@ -17,19 +17,17 @@ on: description: 'Specific container image to use for build (will override the default of "nextstrain build")' required: false -env: - TRIAL_NAME: ${{ github.event.inputs.trial_name }} - NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} - jobs: - gisaid: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - - - name: Launch build + nextstrain-country: + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + env: | + TRIAL_NAME: ${{ github.event.inputs.trial_name }} + NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} run: | set -x @@ -46,7 +44,6 @@ jobs: fi nextstrain build \ - --aws-batch \ --detach \ --cpus 72 \ --memory 140GiB \ @@ -55,35 +52,4 @@ jobs: upload \ --config "${config[@]}" \ --profile nextstrain_profiles/nextstrain-country \ - --set-threads tree=8 \ - |& tee build-launch.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - name: Build info - run: | - if [[ "$TRIAL_NAME" ]]; then - echo "--> Trial name is: $TRIAL_NAME" - echo - echo "--> When completed, the following will be available:" - echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/" - echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time" - else - echo "--> GISAID phylogenetic analysis rebuilding on AWS" - echo - echo "--> When completed, the following will be updated:" - echo "build files: s3://nextstrain-ncov-private/REGION_NAME/" - echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time" - fi - echo - echo "--> You can attach to this AWS job via:" - tail -n1 build-launch.log - echo - JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View this job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo + --set-threads tree=8 diff --git a/.github/workflows/rebuild-gisaid-21L.yml b/.github/workflows/rebuild-gisaid-21L.yml index e80191377..180116e2a 100644 --- a/.github/workflows/rebuild-gisaid-21L.yml +++ b/.github/workflows/rebuild-gisaid-21L.yml @@ -17,19 +17,17 @@ on: description: 'Specific container image to use for build (will override the default of "nextstrain build")' required: false -env: - TRIAL_NAME: ${{ github.event.inputs.trial_name }} - NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} - jobs: gisaid-21L: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - - - name: Launch build + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + env: | + TRIAL_NAME: ${{ github.event.inputs.trial_name }} + NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} run: | set -x @@ -46,7 +44,6 @@ jobs: fi nextstrain build \ - --aws-batch \ --detach \ --cpus 72 \ --memory 140GiB \ @@ -55,35 +52,4 @@ jobs: upload \ --config "${config[@]}" \ --profile nextstrain_profiles/nextstrain-gisaid-21L \ - --set-threads tree=8 \ - |& tee build-launch.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - name: Build info - run: | - if [[ "$TRIAL_NAME" ]]; then - echo "--> Trial name is: $TRIAL_NAME" - echo - echo "--> When completed, the following will be available:" - echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/" - echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/global/all-time" - else - echo "--> GISAID 21L phylogenetic analysis rebuilding on AWS" - echo - echo "--> When completed, the following will be updated:" - echo "build files: s3://nextstrain-ncov-private/REGION_NAME/" - echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/21L/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/ncov/gisaid/21L/global/all-time" - fi - echo - echo "--> You can attach to this AWS job via:" - tail -n1 build-launch.log - echo - JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View this job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo + --set-threads tree=8 diff --git a/.github/workflows/rebuild-gisaid.yml b/.github/workflows/rebuild-gisaid.yml index 1c8cb01fd..baa52d427 100644 --- a/.github/workflows/rebuild-gisaid.yml +++ b/.github/workflows/rebuild-gisaid.yml @@ -17,19 +17,17 @@ on: description: 'Specific container image to use for build (will override the default of "nextstrain build")' required: false -env: - TRIAL_NAME: ${{ github.event.inputs.trial_name }} - NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} - jobs: gisaid: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - - - name: Launch build + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + env: | + TRIAL_NAME: ${{ github.event.inputs.trial_name }} + NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} run: | set -x @@ -46,7 +44,6 @@ jobs: fi nextstrain build \ - --aws-batch \ --detach \ --cpus 72 \ --memory 140GiB \ @@ -55,35 +52,4 @@ jobs: upload \ --config "${config[@]}" \ --profile nextstrain_profiles/nextstrain-gisaid \ - --set-threads tree=8 \ - |& tee build-launch.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - name: Build info - run: | - if [[ "$TRIAL_NAME" ]]; then - echo "--> Trial name is: $TRIAL_NAME" - echo - echo "--> When completed, the following will be available:" - echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/" - echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time" - else - echo "--> GISAID phylogenetic analysis rebuilding on AWS" - echo - echo "--> When completed, the following will be updated:" - echo "build files: s3://nextstrain-ncov-private/REGION_NAME/" - echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time" - fi - echo - echo "--> You can attach to this AWS job via:" - tail -n1 build-launch.log - echo - JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View this job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo + --set-threads tree=8 diff --git a/.github/workflows/rebuild-open.yml b/.github/workflows/rebuild-open.yml index fa808c9fd..ed8415659 100644 --- a/.github/workflows/rebuild-open.yml +++ b/.github/workflows/rebuild-open.yml @@ -18,19 +18,17 @@ on: description: 'Specific container image to use for build (will override the default of "nextstrain build")' required: false -env: - TRIAL_NAME: ${{ github.event.inputs.trial_name }} - NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} - jobs: open: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - - - name: Launch build + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + env: | + TRIAL_NAME: ${{ github.event.inputs.trial_name }} + NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} run: | set -x @@ -47,7 +45,6 @@ jobs: fi nextstrain build \ - --aws-batch \ --detach \ --cpus 72 \ --memory 140GiB \ @@ -56,35 +53,4 @@ jobs: upload \ --config "${config[@]}" \ --profile nextstrain_profiles/nextstrain-open \ - --set-threads tree=8 \ - |& tee build-launch.log - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - name: Build info - run: | - if [[ "$TRIAL_NAME" ]]; then - echo "--> Trial name is: $TRIAL_NAME" - echo - echo "--> When completed, the following will be available:" - echo "build files: s3://nextstrain-staging/files/ncov/open/trial/$TRIAL_NAME/" - echo "nextstrain URLs: https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/global/all-time" - else - echo "--> Open (GenBank) phylogenetic analysis rebuilding on AWS" - echo - echo "--> When completed, the following will be updated:" - echo "build files: s3://nextstrain-data/files/ncov/open/REGION_NAME/" - echo "nextstrain URLs: https://nextstrain.org/ncov/open/REGION_NAME/TIME_SPAN" - echo "e.g. https://nextstrain.org/ncov/open/global/all-time" - fi - echo - echo "--> You can attach to this AWS job via:" - tail -n1 build-launch.log - echo - JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) - echo "--> View this job in the AWS console via" - echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" - echo + --set-threads tree=8