A0-4265 non parity db tests (#1745)

# Description Add two tests for RocksDB snapshots and marking the newest working snapshot as latest. ## Type of change Please delete options that are not relevant. - New feature (non-breaking change which adds functionality) # Checklist: * [Sync from snapshot, Testnet, ParityDB pruned](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9254830988) - OK * [Sync from snapshot, Mainnet, ParityDB pruned](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9264746877) - OK * [Sync from snapshot test, Testnet, ParityDB non-pruned](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9255482291) - OK * [Sync from snapshot test, Mainnet, ParityDB non-pruned](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9255482289) - OK * [Sync from snapshot test, Testnet, RocksDB](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9255482283) - OK * [Sync from snapshot test, Mainnet, RocksDB](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9264534797) - OK
Cardinal-Cryptography · May 28, 2024 · 1939835 · 1939835
1 parent afbd11d
commit 1939835
Show file tree

Hide file tree

Showing 12 changed files with 486 additions and 160 deletions.
diff --git a/.github/actions/sync-from-snapshot/action.yml b/.github/actions/sync-from-snapshot/action.yml
@@ -0,0 +1,77 @@
+---
+name: Sync from snapshot test
+description: This workflow performs live chain (Mainnet or Testnet) sync from a db snapshot.
+inputs:
+  aleph-node-artifact-name:
+    required: true
+    description: 'Name of artifact aleph-node binary'
+  args:
+    description: "Test arguments passed to bash test runner"
+    type: string
+    required: true
+  snapshot-day:
+    description: "Day in date format %Y-%m-%d. If not given, current date is assumed."
+    type: string
+    required: false
+  aws-access-key-id:
+    description: 'AWS Access Key ID to be used when storing artifacts'
+    required: true
+    type: string
+  aws-secret-access-key:
+    description: 'AWS Secret Access Key to be used when storing artifacts'
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Download aleph-node from artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.aleph-node-artifact-name }}
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      env:
+        AWS_REGION: us-east-1
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ env.AWS_REGION }}
+
+    - name: Get snapshot day
+      id: get-snapshot-day
+      shell: bash
+      run: |
+        if [[ '${{ inputs.snapshot-day }}' == "" ]]; then
+          snapshot_day=$(date "+%Y-%m-%d")
+        else
+          snapshot_day='${{ inputs.snapshot-day }}'
+        fi
+        echo "snapshot-day=${snapshot_day}" >> $GITHUB_OUTPUT
+
+    - name: Download snapshot and test db sync
+      shell: bash
+      run: |
+        ./.github/scripts/test_db_sync.sh \
+          ${{ inputs.args }} \
+          --snapshot-day ${{ steps.get-snapshot-day.outputs.snapshot-day }} \
+          --mark-snapshot-as-latest
+
+    - name: Archive logs from failed test
+      if: ${{ failure() }}
+      shell: bash
+      run: |
+        tar -czvf aleph-node-logs.tar.gz running/aleph-node.log
+
+    - name: Upload logs from failed e2e test
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        path: aleph-node-logs.tar.gz
+        name: aleph-node-logs.tar.gz
+        if-no-files-found: error
+        retention-days: 7
diff --git a/.github/scripts/test_parity_db_sync.sh → .github/scripts/test_db_sync.sh b/.github/scripts/test_parity_db_sync.sh → .github/scripts/test_db_sync.sh
@@ -1,11 +1,20 @@
 #!/bin/bash
-set -euo pipefail
-echo "Starting Parity DB sync test."
+set -euox pipefail
+echo "Starting db sync test."
 
+PARITY_DB="false"
 PRUNING="false"
+ENV="mainnet"
+SNAPSHOT_DAY=""
+MARK_SNAPSHOT_AS_LATEST=""
+S3_BUCKET=""
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
+        --parity-db)
+            echo "Using ParityDB."
+            PARITY_DB="true"
+            shift;;
         --pruned)
             echo "Using pruned DB."
             PRUNING="true"
@@ -16,43 +25,69 @@ while [[ $# -gt 0 ]]; do
         --testnet)
             ENV="testnet"
             shift;;
+        --snapshot-day)
+            SNAPSHOT_DAY="$2"
+            shift; shift
+            ;;
+        --mark-snapshot-as-latest)
+            MARK_SNAPSHOT_AS_LATEST="true"
+            shift;;
         *)
             echo "Unrecognized argument: $1"
             exit 1;;
     esac
 done
 
+if [[ "${PRUNING}" == "true" && "${PARITY_DB}" == "false" ]]; then
+    echo "Error! Passed '--pruned' without '--parity-db'"
+    echo "That is an unsupported argument combination."
+    exit 1
+fi
+
 BASE_PATH="running/"
 CHAINSPEC="${BASE_PATH}/chainspec.json"
-DB_ARG="--database paritydb"
 TOP_BLOCK_SCRIPT="./.github/scripts/get_top_block.py"
 
 if [[ "${ENV}" == "mainnet" ]]; then
     SOURCE_CHAINSPEC="./bin/node/src/resources/mainnet_chainspec.json"
     BOOT_NODES=/dns4/bootnode-eu-central-1-0.azero.dev/tcp/30333/p2p/12D3KooWEF1Eo7uFZWdqFsTPP7CehpRt5NeXFwCe3157qpoU5aqd/dns4/bootnode-eu-west-1-0.azero.dev/tcp/30333/p2p/12D3KooWPhi8Qvzvc8iJ4CeQj2vptjc5FTrodKPmra1NS1qfftjr/dns4/bootnode-eu-west-2-0.azero.dev/tcp/30333/p2p/12D3KooWDfUzU64WURE77tXYM9H94xQFAEL6ULQYhzegKsZXjEkC/dns4/bootnode-us-east-1-0.azero.dev/tcp/30333/p2p/12D3KooWFQSGvQii2gRGB5T4M6TXhM83JV4bTEhubCBpdoR6Rkwk/dns4/bootnode-us-east-2-0.azero.dev/tcp/30333/p2p/12D3KooWJnEGVVmnXhVNxV6KWe3EsDPNvPFNcYbQ6amFVGECVAGB
     DB_PATH="chains/mainnet/"
     TARGET_CHAIN="wss://ws.azero.dev"
+    S3_URL="http://db.azero.dev.s3-website.eu-central-1.amazonaws.com"
+    S3_BUCKET="db.azero.dev"
 else
     SOURCE_CHAINSPEC="./bin/node/src/resources/testnet_chainspec.json"
     BOOT_NODES=/dns4/bootnode-eu-central-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWRkGLz4YbVmrsWK75VjFTs8NvaBu42xhAmQaP4KeJpw1L/dns4/bootnode-eu-west-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWFVXnvJdPuGnGYMPn5qLQAQYwmRBgo6SmEQsKZSrDoo2k/dns4/bootnode-eu-west-2-0.test.azero.dev/tcp/30333/p2p/12D3KooWAkqYFFKMEJn6fnPjYnbuBBsBZq6fRFJZYR6rxnuCZWCC/dns4/bootnode-us-east-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWQFkkFr5aM5anGEiUCQiGUdRyWgrdpvSjBgWAUS9srLE4/dns4/bootnode-us-east-2-0.test.azero.dev/tcp/30333/p2p/12D3KooWD5s2dkifJua69RbLwEREDdJjsNHvavNRGxdCvzhoeaLc
     DB_PATH="chains/testnet/"
     TARGET_CHAIN="wss://ws.test.azero.dev"
+    S3_URL="http://db.test.azero.dev.s3-website.eu-central-1.amazonaws.com"
+    S3_BUCKET="db.test.azero.dev"
+fi
+
+declare -a DB_ARG
+S3_SNAPSHOT_PREFIX=""
+LATEST_SNAPSHOT_NAME=""
+if [[ "${PARITY_DB}" == "true" ]]; then
+    DB_ARG+=("--database paritydb")
+    S3_SNAPSHOT_PREFIX="db_backup_parity"
+    LATEST_SNAPSHOT_NAME="latest-parity.html"
 fi
 if [[ "${PRUNING}" == "true" ]]; then
-    DB_ARG="--enable-pruning"
-    if [[ "${ENV}" == "mainnet" ]]; then
-        DB_SNAPSHOT_URL="http://db.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity-pruned.html"
-    else
-        DB_SNAPSHOT_URL="http://db.test.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity-pruned.html"
-    fi
-else
-    if [[ "${ENV}" == "mainnet" ]]; then
-       DB_SNAPSHOT_URL="http://db.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity.html"
-    else
-       DB_SNAPSHOT_URL="http://db.test.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity.html"
-    fi
+    DB_ARG+=("--enable-pruning")
+    S3_SNAPSHOT_PREFIX="db_backup_parity_pruned"
+    LATEST_SNAPSHOT_NAME="latest-parity-pruned.html"
+fi
+if [[ "${PARITY_DB}" != "true" && "${PRUNING}" != "true" ]]; then
+    S3_SNAPSHOT_PREFIX="db_backup"
+    LATEST_SNAPSHOT_NAME="latest.html"
+fi
+
+if [[ -z "${SNAPSHOT_DAY}" ]]; then
+    SNAPSHOT_DAY=$(date "+%Y-%m-%d")
 fi
 
+DB_SNAPSHOT_URL="${S3_URL}/${SNAPSHOT_DAY}/${S3_SNAPSHOT_PREFIX}_${SNAPSHOT_DAY}.tar.gz"
+
 initialize() {
     pip install substrate-interface
     mkdir -p "${BASE_PATH}"
@@ -64,13 +99,11 @@ get_snapshot () {
     mkdir -p "${DB_SNAPSHOT_PATH}"
     pushd "${DB_SNAPSHOT_PATH}" > /dev/null
 
-    set +e
-    wget -q -O - ${DB_SNAPSHOT_URL} | tar xzf -
+    wget -q -O - "${DB_SNAPSHOT_URL}" | tar xzf -
     if [[ 0 -ne $? ]]
     then
         error "Failed to download and unpack the snapshot."
     fi
-    set -e
     popd > /dev/null
 }
 
@@ -103,7 +136,7 @@ chmod +x aleph-node
     --name sync-from-snapshot-tester \
     --bootnodes "${BOOT_NODES}" \
     --node-key-file "${BASE_PATH}/p2p_secret" \
-    ${DB_ARG} \
+    ${DB_ARG[*]} \
     --no-mdns 1>/dev/null 2> "${BASE_PATH}/aleph-node.log" &
 
 get_current_block
@@ -114,3 +147,10 @@ while [ $CURRENT_BLOCK -le $TARGET_BLOCK ]; do
     get_current_block
     echo "Sync status: ${CURRENT_BLOCK}/${TARGET_BLOCK}".
 done
+
+if [[ "${MARK_SNAPSHOT_AS_LATEST}" == "true" ]]; then
+  echo "<meta http-equiv=\"refresh\" content=\"0;url=${S3_URL}/${SNAPSHOT_DAY}/${S3_SNAPSHOT_PREFIX}_${SNAPSHOT_DAY}.tar.gz\">" | \
+    aws s3 cp - "s3://${S3_BUCKET}/${LATEST_SNAPSHOT_NAME}" \
+      --website-redirect "${S3_URL}/${SNAPSHOT_DAY}/${S3_SNAPSHOT_PREFIX}_${SNAPSHOT_DAY}.tar.gz"
+fi
+
diff --git a/.github/workflows/_check-vars-and-secrets.yml b/.github/workflows/_check-vars-and-secrets.yml
@@ -51,6 +51,10 @@ jobs:
             -z '${{ secrets.DOCKERHUB_PASSWORD }}' || \
             -z '${{ secrets.DOCKERHUB_USERNAME }}' || \
             -z '${{ secrets.SLACK_WEBHOOK_TRACK_APPLICATIONS }}' || \
+            -z '${{ secrets.AWS_TESTNET_S3_ACCESS_KEY_ID }}' || \
+            -z '${{ secrets.AWS_TESTNET_S3_SECRET_ACCESS_KEY }}' || \
+            -z '${{ secrets.AWS_MAINNET_S3_ACCESS_KEY_ID }}' || \
+            -z '${{ secrets.AWS_MAINNET_S3_SECRET_ACCESS_KEY }}' || \
             -z '${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }}'
           ]]; then
             echo '!!! Some repository secrets are either missing or empty.'

diff --git a/.github/workflows/_sync-from-snapshot.yml b/.github/workflows/_sync-from-snapshot.yml
diff --git a/.github/workflows/sync-from-snapshot-mainnet-paritydb-pruned.yml b/.github/workflows/sync-from-snapshot-mainnet-paritydb-pruned.yml
@@ -7,20 +7,59 @@
 
 name: Sync from snapshot, Mainnet, ParityDB pruned
 on:
-  # At 03:00 on Wednesday and Saturday
+  # At 20:00 on Wednesday and Saturday
+  # Time corresponds with a snapshot creation time
   schedule:
-    - cron: '0 3 * * 3,6'
+    - cron: '0 20 * * 3,6'
   workflow_dispatch:
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: false
 
 jobs:
-  main:
-    name: Test sync
-    uses: ./.github/workflows/_sync-from-snapshot.yml
+  check-vars-and-secrets:
+    name: Check vars and secrets
+    uses: ./.github/workflows/_check-vars-and-secrets.yml
     secrets: inherit
+
+  build-production-aleph-node:
+    needs: [check-vars-and-secrets]
+    name: Build production aleph-node
+    uses: ./.github/workflows/_build-aleph-node.yml
     with:
-      timeout: 60
-      args: --mainnet --pruned
+      ref: ${{ github.ref }}
+      production: true
+
+  test-sync:
+    needs: [build-production-aleph-node]
+    name: Download snapshot and run
+    runs-on: [self-hosted, Linux, X64, medium-1000GB]
+    timeout-minutes: 60
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Test sync
+        uses: ./.github/actions/sync-from-snapshot
+        with:
+          # yamllint disable-line rule:line-length
+          aleph-node-artifact-name: ${{ needs.build-production-aleph-node.outputs.artifact-name-binary }}
+          args: --mainnet --parity-db --pruned
+          aws-access-key-id: ${{ secrets.AWS_MAINNET_S3_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_MAINNET_S3_SECRET_ACCESS_KEY }}
+
+  slack-notification:
+    name: Slack notification
+    runs-on: ubuntu-20.04
+    needs: [test-sync]
+    if: >
+      !cancelled() &&
+      github.event_name != 'workflow_dispatch'
+    steps:
+      - name: Send Slack message
+        uses: Cardinal-Cryptography/github-actions/slack-notification@v7
+        with:
+          notify-on: "failure"
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }}