Merge PR #58: Prebuild docker testnet

* solidity docker ci * integration tests * syntax issue * only run on conclusion of other events * rename gravity * update test to use binaries * copy script * nexe is too slow * check for binary * don't push, fill cache * set up build contexts * build image * work around context * clean docker ignore * workflow run * remove workflow run filter * hail mary * different action * extend timeout * only workflow run * all branches * syntax * single workflow * no matrix * syntax * check outputs * echo * debugging * debugging * pull image * two step * retag * procure all images * prebuilt test * matrix of tests * invalid file name * log output * copy over contracts * don't prebuild solidity * inspect contract deployer * stringify * don't precompile node * don't pull solidity * check for stderr * only happy path for now * log err * add bash to help debug * add compatibility layer * don't use alpine * use original dockerfile * add timeout hack * expose test runner logs * build test runner * to string * prebuild test runner * dependency * more logs * docker cacheing * step naming * check for fast failing orchestrator * info logs only * don't copy over target * verbose test * enable all tests * unneeded comments * check auto remove env * test workflows in their own place * naming * only happy paths * import naming * remove check that only works in happy path * prebuild contract deployer * cache docker layers for go and node * test more tests * more metadata setup * in both tests * remove failing tests * don't check out put * check container code * disable v2 * disable happy path * batch stress doesnt work
PeggyJV · Jul 9, 2021 · 493cb6f · 493cb6f
1 parent c9d3720
commit 493cb6f
Show file tree

Hide file tree

Showing 12 changed files with 552 additions and 252 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -1,55 +1,30 @@
-name: Go
+name: Go tests
 
 on:
   push:
     branches:
       - main
   pull_request:
 
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
 jobs:
-  go-build:
+  go-test:
     permissions:
       contents: read
       packages: write
     runs-on: ubuntu-latest
     steps:
-      - name: install-go
+      - name: Install Go
         uses: actions/setup-go@v2
         with:
           go-version: ^1.16
-      - name: checkout
+      - name: Checkout Branch
         uses: actions/checkout@v2
-      - name: go-cache
+      - name: Create Go cache
         uses: actions/cache@v2
-        with: 
+        with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('module/go.sum') }}
-      - name: go-test
-        run: cd module && make test
-      - name: go-build
-        run: cd module && make build
-      - name: container-login
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: labels
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-gravity
-      - name: build-and-push
-        uses: docker/build-push-action@v2
-        with:
-          context: ./module
-          file: module/ci.Dockerfile
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+      - name: Run Go tests
+        run: cd module && make test
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -2,54 +2,270 @@ name: Integration tests
 
 on:
   push:
-    branches: [ master, main ]
+    branches:
+      - main
   pull_request:
-    branches: [ master, main ]
+
+env:
+  CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  happy-path:
+  rust-build:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
+      - name: Checkout branch
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        id: docker-cache
         with:
-          go-version: ^1.16
-      - uses: actions/checkout@v2
-      - name: Run all up happy-path test
-        run: cd testnet && go test -c && cd .. && ./testnet/testnet.test -test.run BasicChainDynamicKeys -test.failfast
-  validator-out:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run all up test with a validator out
-        run: make e2e_validator_out
-  valset-stress:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run all up valset stress test
-        run: make e2e_valset_stress
-  batch-stress:
+          path: /tmp/.buildx-cache
+          # Key is named differently to avoid collision
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
+      - name: Container service login
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-orchestrator
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./orchestrator
+          builder: ${{ steps.buildx.outputs.name }}
+          file: orchestrator/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+      - name: Move Docker cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  test-runner-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Run all up batch stress test
-        run: make e2e_batch_stress
-  v2-happy-path:
+      - name: Checkout branch
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        id: docker-cache
+        with:
+          path: /tmp/.buildx-cache
+          # Key is named differently to avoid collision
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
+      - name: Container service login
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-test-runner
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./orchestrator
+          builder: ${{ steps.buildx.outputs.name }}
+          file: orchestrator/testnet.Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+      - name: Move Docker cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  go-build:
+    permissions:
+      contents: read
+      packages: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Run all up batch stress test
-        run: make e2e_v2_happy_path
-  arbitrary-logic:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        id: docker-cache
+        with:
+          path: /tmp/.buildx-cache
+          # Key is named differently to avoid collision
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
+      - name: container-login
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: labels
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-gravity
+      - name: build-and-push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./module
+          file: module/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          builder: ${{ steps.buildx.outputs.name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+      - name: Move Docker cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  solidity-build:
+    permissions:
+      contents: read
+      packages: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Run all up arbitrary logic test
-        run: make e2e_arbitrary_logic
-  orchestrator-keys:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        id: docker-cache
+        with:
+          path: /tmp/.buildx-cache
+          # Key is named differently to avoid collision
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
+      - name: container-login
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: labels
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-solidity
+      - name: build-and-push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./solidity
+          file: solidity/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          builder: ${{ steps.buildx.outputs.name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+      - name: Move Docker cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  integration-tests:
     runs-on: ubuntu-latest
+    needs: [ rust-build, go-build, test-runner-build, solidity-build ]
+    strategy:
+      matrix:
+        test_type: [
+          # working tests
+            "ORCHESTRATOR_KEYS",
+            "ARBITRARY_LOGIC",
+
+          # non-working tests
+#            "VALIDATOR_OUT",
+#            "BATCH_STRESS",
+#            "VALSET_STRESS",
+#            "HAPPY_PATH",
+#            "V2_HAPPY_PATH",
+        ]
     steps:
-      - uses: actions/checkout@v2
-      - name: Run all up arbitrary logic test
-        run: make e2e_orchestrator_keys
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: go-cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('testnet/go.sum') }}
+      - name: container-login
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: docker gravity metadata
+        id: meta-gravity
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-gravity
+      - name: docker orchestrator metadata
+        id: meta-orchestrator
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-orchestrator
+      - name: docker test runner metadata
+        id: meta-test-runner
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-test-runner
+      - name: docker contract deployer metadata
+        id: meta-solidity
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-solidity
+      - name: pull gravity image
+        run: docker pull ${{ steps.meta-gravity.outputs.tags }}
+      - name: rename gravity image
+        run: docker tag ${{ steps.meta-gravity.outputs.tags }} gravity:prebuilt
+      - name: pull orchestrator image
+        run: docker pull ${{ steps.meta-orchestrator.outputs.tags }}
+      - name: rename orchestrator image
+        run: docker tag ${{ steps.meta-gravity.outputs.tags }} orchestrator:prebuilt
+      - name: pull test-runner image
+        run: docker pull ${{ steps.meta-test-runner.outputs.tags }}
+      - name: rename test-runner image
+        run: docker tag ${{ steps.meta-test-runner.outputs.tags }} test-runner:prebuilt
+      - name: pull contract deployer image
+        run: docker pull ${{ steps.meta-solidity.outputs.tags }}
+      - name: rename contract deployer image
+        run: docker tag ${{ steps.meta-solidity.outputs.tags }} solidity:prebuilt
+      - name: Run all up ci test
+        run: cd testnet && go test -c && cd .. && ./testnet/testnet.test -test.run PrebuiltCi -test.failfast -test.v
+        env:
+          TEST_TYPE: ${{ matrix.test_type}}
diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
@@ -9,6 +9,11 @@ on:
   pull_request:
     branches: [master, main]
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+
 jobs:
   node-build:
     runs-on: ubuntu-latest
@@ -28,10 +33,7 @@ jobs:
       - run: cd solidity && npm run typechain
       - run: cd solidity && npm run evm &
       - run: cd solidity && npm run test
-      - run: cd solidity && npm run compile-deployer
-      - run: npm i nexe -g
-      - run: cd solidity && nexe dist/contract-deployer.js --build --resource "./dist/**/*"
-
+
   arbitrary-logic-tests:
     runs-on: ubuntu-latest
     strategy: