Skip to content

Commit

Permalink
Use pathogen-repo-build in GH Action workflows
Browse files Browse the repository at this point in the history
Updates all GH Action workflows for running `nextstrain build` to use
the shared `pathogen-repo-build` workflow.

Resolves #1113
  • Loading branch information
joverlee521 committed Jun 26, 2024
1 parent eb488d6 commit c30b752
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 226 deletions.
67 changes: 18 additions & 49 deletions .github/workflows/rebuild-100k.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,80 +9,49 @@ on:

jobs:
gisaid:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4

- uses: nextstrain/.github/actions/setup-nextstrain-cli@master

- name: Launch GISAID build
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
set -x
declare -a config
config+=(slack_token=$SLACK_TOKEN)
nextstrain build \
--aws-batch \
--detach \
--cpus 16 \
--memory 31GiB \
. \
upload \
--configfile nextstrain_profiles/100k/config-gisaid.yaml \
--config "${config[@]}" \
--set-threads tree=8 \
|& tee build-launch-gisaid.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

- name: Launch open build
--set-threads tree=8
artifact-name: gisaid-build-output

open:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
set -x
declare -a config
config+=(slack_token=$SLACK_TOKEN)
nextstrain build \
--aws-batch \
--detach \
--cpus 16 \
--memory 31GiB \
. \
upload \
--configfile nextstrain_profiles/100k/config-open.yaml \
--config "${config[@]}" \
--set-threads tree=8 \
|& tee build-launch-open.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}


- name: Build info
run: |
echo "--> 100k samples for GISAID + Open data rebuilding (using separate AWS jobs)"
echo
echo "--> When completed, the following files will be updated:"
echo "s3://nextstrain-data/files/ncov/open/100k/metadata.tsv.xz"
echo "s3://nextstrain-data/files/ncov/open/100k/sequences.fasta.xz"
echo "s3://nextstrain-ncov-private/100k/metadata.tsv.xz"
echo "s3://nextstrain-ncov-private/100k/sequences.fasta.xz"
echo
echo "--> You can attach to the GISAID AWS job via:"
tail -n1 build-launch-gisaid.log
echo
echo "--> You can attach to the Open AWS job via:"
tail -n1 build-launch-open.log
echo
JOBID=$( tail -n1 build-launch-gisaid.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View the GISAID job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
JOBID=$( tail -n1 build-launch-open.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View the Open job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
--set-threads tree=8
artifact-name: open-build-output
56 changes: 11 additions & 45 deletions .github/workflows/rebuild-country.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,17 @@ on:
description: 'Specific container image to use for build (will override the default of "nextstrain build")'
required: false

env:
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}

jobs:
gisaid:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4

- uses: nextstrain/.github/actions/setup-nextstrain-cli@master

- name: Launch build
nextstrain-country:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
run: |
set -x
Expand All @@ -46,7 +44,6 @@ jobs:
fi
nextstrain build \
--aws-batch \
--detach \
--cpus 72 \
--memory 140GiB \
Expand All @@ -55,35 +52,4 @@ jobs:
upload \
--config "${config[@]}" \
--profile nextstrain_profiles/nextstrain-country \
--set-threads tree=8 \
|& tee build-launch.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

- name: Build info
run: |
if [[ "$TRIAL_NAME" ]]; then
echo "--> Trial name is: $TRIAL_NAME"
echo
echo "--> When completed, the following will be available:"
echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time"
else
echo "--> GISAID phylogenetic analysis rebuilding on AWS"
echo
echo "--> When completed, the following will be updated:"
echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time"
fi
echo
echo "--> You can attach to this AWS job via:"
tail -n1 build-launch.log
echo
JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View this job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
--set-threads tree=8
54 changes: 10 additions & 44 deletions .github/workflows/rebuild-gisaid-21L.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,17 @@ on:
description: 'Specific container image to use for build (will override the default of "nextstrain build")'
required: false

env:
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}

jobs:
gisaid-21L:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4

- uses: nextstrain/.github/actions/setup-nextstrain-cli@master

- name: Launch build
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
run: |
set -x
Expand All @@ -46,7 +44,6 @@ jobs:
fi
nextstrain build \
--aws-batch \
--detach \
--cpus 72 \
--memory 140GiB \
Expand All @@ -55,35 +52,4 @@ jobs:
upload \
--config "${config[@]}" \
--profile nextstrain_profiles/nextstrain-gisaid-21L \
--set-threads tree=8 \
|& tee build-launch.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

- name: Build info
run: |
if [[ "$TRIAL_NAME" ]]; then
echo "--> Trial name is: $TRIAL_NAME"
echo
echo "--> When completed, the following will be available:"
echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/global/all-time"
else
echo "--> GISAID 21L phylogenetic analysis rebuilding on AWS"
echo
echo "--> When completed, the following will be updated:"
echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/21L/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/ncov/gisaid/21L/global/all-time"
fi
echo
echo "--> You can attach to this AWS job via:"
tail -n1 build-launch.log
echo
JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View this job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
--set-threads tree=8
54 changes: 10 additions & 44 deletions .github/workflows/rebuild-gisaid.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,17 @@ on:
description: 'Specific container image to use for build (will override the default of "nextstrain build")'
required: false

env:
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}

jobs:
gisaid:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4

- uses: nextstrain/.github/actions/setup-nextstrain-cli@master

- name: Launch build
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
run: |
set -x
Expand All @@ -46,7 +44,6 @@ jobs:
fi
nextstrain build \
--aws-batch \
--detach \
--cpus 72 \
--memory 140GiB \
Expand All @@ -55,35 +52,4 @@ jobs:
upload \
--config "${config[@]}" \
--profile nextstrain_profiles/nextstrain-gisaid \
--set-threads tree=8 \
|& tee build-launch.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

- name: Build info
run: |
if [[ "$TRIAL_NAME" ]]; then
echo "--> Trial name is: $TRIAL_NAME"
echo
echo "--> When completed, the following will be available:"
echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time"
else
echo "--> GISAID phylogenetic analysis rebuilding on AWS"
echo
echo "--> When completed, the following will be updated:"
echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time"
fi
echo
echo "--> You can attach to this AWS job via:"
tail -n1 build-launch.log
echo
JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View this job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
--set-threads tree=8
54 changes: 10 additions & 44 deletions .github/workflows/rebuild-open.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,17 @@ on:
description: 'Specific container image to use for build (will override the default of "nextstrain build")'
required: false

env:
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}

jobs:
open:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4

- uses: nextstrain/.github/actions/setup-nextstrain-cli@master

- name: Launch build
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
TRIAL_NAME: ${{ github.event.inputs.trial_name }}
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
run: |
set -x
Expand All @@ -47,7 +45,6 @@ jobs:
fi
nextstrain build \
--aws-batch \
--detach \
--cpus 72 \
--memory 140GiB \
Expand All @@ -56,35 +53,4 @@ jobs:
upload \
--config "${config[@]}" \
--profile nextstrain_profiles/nextstrain-open \
--set-threads tree=8 \
|& tee build-launch.log
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

- name: Build info
run: |
if [[ "$TRIAL_NAME" ]]; then
echo "--> Trial name is: $TRIAL_NAME"
echo
echo "--> When completed, the following will be available:"
echo "build files: s3://nextstrain-staging/files/ncov/open/trial/$TRIAL_NAME/"
echo "nextstrain URLs: https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/global/all-time"
else
echo "--> Open (GenBank) phylogenetic analysis rebuilding on AWS"
echo
echo "--> When completed, the following will be updated:"
echo "build files: s3://nextstrain-data/files/ncov/open/REGION_NAME/"
echo "nextstrain URLs: https://nextstrain.org/ncov/open/REGION_NAME/TIME_SPAN"
echo "e.g. https://nextstrain.org/ncov/open/global/all-time"
fi
echo
echo "--> You can attach to this AWS job via:"
tail -n1 build-launch.log
echo
JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
echo "--> View this job in the AWS console via"
echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
echo
--set-threads tree=8

0 comments on commit c30b752

Please sign in to comment.