feat: self hosted e2e tests (osmosis-labs#5951)

* attempt self hosted e2e tests

* prune docker containers

* filter E2E docker container pruning

* fix syntax

* syntax

* syntax

* syntax

* use mutex to prevent concurrent map writes

* stream logs

* use wg for run validators

* various perf changes

* make a standard timeout for all tests

.github/workflows/test.yml

@@ -12,7 +12,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: '1.20.5'
+  GO_VERSION: "1.20.5"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -22,11 +22,9 @@ jobs:
   get_diff:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Check out repository code
+      - name: Check out repository code
         uses: actions/checkout@v3
-      -
-        name: Get git diff
+      - name: Get git diff
         uses: technote-space/[email protected]
         with:
           PATTERNS: |
@@ -45,37 +43,29 @@ jobs:
     if: needs.get_diff.outputs.git_diff
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Check out repository code
+      - name: Check out repository code
         uses: actions/checkout@v3
-      -
-        name: 🐿 Setup Golang
+      - name: 🐿 Setup Golang
         uses: actions/setup-go@v4
         with:
           go-version: ${{env.GO_VERSION}}
-      -
-        name: Create a file with all core Cosmos SDK pkgs
+      - name: Create a file with all core Cosmos SDK pkgs
         run: go list ./... ./osmomath/... ./osmoutils/... ./x/ibc-hooks/... ./x/epochs | grep -E -v 'tests/simulator|e2e' > pkgs.txt
-      -
-        name: Split pkgs into 4 files
+      - name: Split pkgs into 4 files
         run: split -d -n l/4 pkgs.txt pkgs.txt.part.
-      -
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
         with:
           name: "${{ github.sha }}-00"
           path: ./pkgs.txt.part.00
-      -
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
         with:
           name: "${{ github.sha }}-01"
           path: ./pkgs.txt.part.01
-      -
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
         with:
           name: "${{ github.sha }}-02"
           path: ./pkgs.txt.part.02
-      -
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
         with:
           name: "${{ github.sha }}-03"
           path: ./pkgs.txt.part.03
@@ -89,23 +79,18 @@ jobs:
       matrix:
         part: ["00", "01", "02", "03"]
     steps:
-      -
-        name: Check out repository code
+      - name: Check out repository code
         uses: actions/checkout@v3
-      -
-        name: 🐿 Setup Golang
+      - name: 🐿 Setup Golang
         uses: actions/setup-go@v4
         with:
           go-version: ${{env.GO_VERSION}}
-      -
-        name: Display go version
+      - name: Display go version
         run: go version
-      -
-        uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v3
         with:
           name: "${{ github.sha }}-${{ matrix.part }}"
-      -
-        name: Test & coverage report creation
+      - name: Test & coverage report creation
         run: |
           VERSION=$(echo $(git describe --tags) | sed 's/^v//') || VERSION=${GITHUB_SHA}
           TESTS=$(cat pkgs.txt.part.${{ matrix.part }})
@@ -115,59 +100,63 @@ jobs:
   e2e:
     needs: get_diff
     if: needs.get_diff.outputs.git_diff
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     timeout-minutes: 25
     steps:
-      -
-        name: Check out repository code
+      - name: Clean up Pre-Existing E2E Docker containers
+        run: |
+          # Remove containers with names starting with "osmo-test-"
+          docker ps -aqf "name=osmo-test-*" | xargs -r docker rm -f
+
+          # Remove containers with names starting with "hermes-relayer"
+          docker ps -aqf "name=hermes-relayer*" | xargs -r docker rm -f
+      - name: Check out repository code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      -
-        name: 🐿 Setup Golang
+      - name: 🐿 Setup Golang
         uses: actions/setup-go@v4
         with:
           go-version: ${{env.GO_VERSION}}
-      -
-        name: Set up QEMU
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      -
-        name: Build e2e image
+      - name: Build e2e image
         uses: docker/build-push-action@v4
         with:
           load: true
           context: .
           tags: osmosis:debug
-          # Use experimental Cache backend API: 
+          # Use experimental Cache backend API:
           # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#cache-backend-api
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |
             BASE_IMG_TAG=debug
-      -
-        name: Test e2e and Upgrade
+      - name: Test e2e and Upgrade
         run: make test-e2e-ci
-      -
-        name: Dump docker logs on failure
+      - name: Dump docker logs on failure
         if: failure()
         uses: jwalton/gh-docker-logs@v2
         with:
           dest: "./logs"
-      -
-        name: Tar logs
+      - name: Tar logs
         if: failure()
         run: tar cvzf ./logs.tgz ./logs
-      -
-        name: Upload logs to GitHub
+      - name: Upload logs to GitHub
         uses: actions/upload-artifact@v3
         with:
           name: logs.tgz
           path: ./logs.tgz
         if: failure()
-      -
-        name: 🧹 Clean up Osmosis Home
+      - name: 🧹 Clean up Osmosis Home
         if: always()
         run: rm -rf $HOME/.osmosisd/ || true
+      - name: Clean up E2E Docker containers
+        run: |
+          # Remove containers with names starting with "osmo-test-"
+          docker ps -aqf "name=osmo-test-*" | xargs -r docker rm -f
+
+          # Remove containers with names starting with "hermes-relayer"
+          docker ps -aqf "name=hermes-relayer*" | xargs -r docker rm -f

Makefile

@@ -282,7 +282,7 @@ run-querygen:
 ###############################################################################
 
 PACKAGES_UNIT=$(shell go list ./... ./osmomath/... ./osmoutils/... ./x/ibc-hooks/... ./x/epochs | grep -E -v 'tests/simulator|e2e')
-PACKAGES_E2E=$(shell go list ./... | grep '/e2e')
+PACKAGES_E2E := $(shell go list ./... | grep '/e2e' | awk -F'/e2e' '{print $$1 "/e2e"}' | uniq)
 PACKAGES_SIM=$(shell go list ./... | grep '/tests/simulator')
 TEST_PACKAGES=./...
 

tests/e2e/configurer/base.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
@@ -75,18 +76,29 @@ func (bc *baseConfigurer) RunValidators() error {
 func (bc *baseConfigurer) runValidators(chainConfig *chain.Config) error {
 	bc.t.Logf("starting %s validator containers...", chainConfig.Id)
 
-	errCh := make(chan error) // Channel to collect errors
+	var wg sync.WaitGroup
+	errCh := make(chan error, len(chainConfig.NodeConfigs)) // Buffer the channel to avoid blocking
+
+	// Increment the WaitGroup counter for each node
+	wg.Add(len(chainConfig.NodeConfigs))
 
 	// Iterate over each node
 	for _, node := range chainConfig.NodeConfigs {
 		go func(n *chain.NodeConfig) {
+			defer wg.Done()  // Decrement the WaitGroup counter when the goroutine is done
 			errCh <- n.Run() // Run the node and send any error to the channel
 		}(node)
 	}
 
-	// Wait for goroutines to finish and collect errors
-	for range chainConfig.NodeConfigs {
-		if err := <-errCh; err != nil {
+	// Wait for all goroutines to finish
+	wg.Wait()
+
+	// Close the error channel since all goroutines are done sending errors
+	close(errCh)
+
+	// Collect errors from the channel
+	for err := range errCh {
+		if err != nil {
 			return err
 		}
 	}
@@ -172,8 +184,8 @@ func (bc *baseConfigurer) runIBCRelayer(chainConfigA *chain.Config, chainConfigB
 
 		return status == "success" && len(chains) == 2
 	},
-		5*time.Minute,
-		time.Second,
+		time.Minute,
+		10*time.Millisecond,
 		"hermes relayer not healthy")
 
 	bc.t.Logf("started Hermes relayer container: %s", hermesResource.Container.ID)

tests/e2e/configurer/chain/chain.go

@@ -193,6 +193,14 @@ func (c *Config) SendIBC(dstChain *Config, recipient string, token sdk.Coin) {
 	_, _, err = c.containerManager.ExecHermesCmd(c.t, cmd, "SUCCESS")
 	require.NoError(c.t, err)
 
+	cmd = []string{"hermes", "clear", "packets", "--chain", dstChain.Id, "--port", "transfer", "--channel", "channel-0"}
+	_, _, err = c.containerManager.ExecHermesCmd(c.t, cmd, "SUCCESS")
+	require.NoError(c.t, err)
+
+	cmd = []string{"hermes", "clear", "packets", "--chain", c.Id, "--port", "transfer", "--channel", "channel-0"}
+	_, _, err = c.containerManager.ExecHermesCmd(c.t, cmd, "SUCCESS")
+	require.NoError(c.t, err)
+
 	require.Eventually(
 		c.t,
 		func() bool {
@@ -211,8 +219,8 @@ func (c *Config) SendIBC(dstChain *Config, recipient string, token sdk.Coin) {
 				return false
 			}
 		},
-		5*time.Minute,
-		time.Second,
+		1*time.Minute,
+		10*time.Millisecond,
 		"tx not received on destination chain",
 	)
 

tests/e2e/configurer/chain/commands.go

@@ -259,6 +259,14 @@ func (n *NodeConfig) SendIBCTransfer(dstChain *Config, from, recipient, memo str
 	_, _, err := n.containerManager.ExecHermesCmd(n.t, cmd, "SUCCESS")
 	require.NoError(n.t, err)
 
+	cmd = []string{"hermes", "clear", "packets", "--chain", dstChain.Id, "--port", "transfer", "--channel", "channel-0"}
+	_, _, err = n.containerManager.ExecHermesCmd(n.t, cmd, "SUCCESS")
+	require.NoError(n.t, err)
+
+	cmd = []string{"hermes", "clear", "packets", "--chain", n.chainId, "--port", "transfer", "--channel", "channel-0"}
+	_, _, err = n.containerManager.ExecHermesCmd(n.t, cmd, "SUCCESS")
+	require.NoError(n.t, err)
+
 	n.LogActionF("successfully submitted sent IBC transfer")
 }
 
@@ -681,8 +689,8 @@ func (n *NodeConfig) SendIBC(dstChain *Config, recipient string, token sdk.Coin)
 				return false
 			}
 		},
-		5*time.Minute,
-		time.Second,
+		time.Minute,
+		10*time.Millisecond,
 		"tx not received on destination chain",
 	)
 
@@ -821,6 +829,6 @@ func (n *NodeConfig) ParamChangeProposal(subspace, key string, value []byte, cha
 			return false
 		}
 		return status == proposalStatusPassed
-	}, time.Minute*30, time.Millisecond*500)
+	}, time.Minute, 10*time.Millisecond)
 	return nil
 }

tests/e2e/configurer/chain/node.go

@@ -72,8 +72,8 @@ func (n *NodeConfig) Run() error {
 			n.t.Logf("started node container: %s", n.Name)
 			return true
 		},
-		2*time.Minute,
-		time.Second,
+		time.Minute,
+		10*time.Millisecond,
 		"Osmosis node failed to produce blocks",
 	)
 

tests/e2e/configurer/chain/queries.go

@@ -231,7 +231,7 @@ func (n *NodeConfig) QueryGRPCGateway(path string, parameters ...string) ([]byte
 		}
 
 		return resp.StatusCode != http.StatusServiceUnavailable
-	}, time.Minute, time.Millisecond*10, "failed to execute HTTP request")
+	}, time.Minute, 10*time.Millisecond, "failed to execute HTTP request")
 
 	defer resp.Body.Close()
 

tests/e2e/containers/containers.go

@@ -121,7 +121,7 @@ func (m *Manager) ExecCmd(t *testing.T, containerName string, command []string,
 		errBuf bytes.Buffer
 	)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 
 	if m.isDebugLogEnabled {
@@ -218,7 +218,7 @@ func (m *Manager) ExecCmd(t *testing.T, containerName string, command []string,
 			return true
 		},
 		time.Minute,
-		50*time.Millisecond,
+		10*time.Millisecond,
 		fmt.Sprintf("success condition (%s) was not met.\nstdout:\n %s\nstderr:\n %s\n",
 			success, outBuf.String(), errBuf.String()),
 	)

tests/e2e/e2e_setup_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"os"
 	"strconv"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
@@ -34,6 +35,7 @@ type IntegrationTestSuite struct {
 	skipUpgrade   bool
 	skipIBC       bool
 	skipStateSync bool
+	mutex         sync.Mutex
 }
 
 func TestIntegrationTestSuite(t *testing.T) {

tests/e2e/e2e_test.go

@@ -1195,7 +1195,7 @@ func (s *IntegrationTestSuite) IBCTokenTransferRateLimiting() {
 		s.Eventually(func() bool {
 			val := chainANode.QueryParams(ibcratelimittypes.ModuleName, string(ibcratelimittypes.KeyContractAddress))
 			return strings.Contains(val, param)
-		}, time.Second*30, time.Millisecond*500)
+		}, time.Second*30, time.Second)
 	}
 }
 
@@ -1228,7 +1228,7 @@ func (s *IntegrationTestSuite) IBCWasmHooks() {
 	// check the balance of the contract
 	denomTrace := transfertypes.ParseDenomTrace(transfertypes.GetPrefixedDenom("transfer", "channel-0", "uosmo"))
 	ibcDenom := denomTrace.IBCDenom()
-	s.CheckBalance(chainANode, contractAddr, ibcDenom, transferAmount)
+	s.CallCheckBalance(chainANode, contractAddr, ibcDenom, transferAmount)
 
 	// sender wasm addr
 	senderBech32, err := ibchookskeeper.DeriveIntermediateSender("channel-0", validatorAddr, "osmo")
@@ -1301,7 +1301,7 @@ func (s *IntegrationTestSuite) PacketForwarding() {
 	chainANode.SendIBCTransfer(chainB, validatorAddr, validatorAddr, string(forwardMemo), coin)
 
 	// check the balance of the contract
-	s.CheckBalance(chainANode, contractAddr, "uosmo", transferAmount)
+	s.CallCheckBalance(chainANode, contractAddr, "uosmo", transferAmount)
 
 	// sender wasm addr
 	senderBech32, err := ibchookskeeper.DeriveIntermediateSender("channel-0", validatorAddr, "osmo")
@@ -1615,8 +1615,8 @@ func (s *IntegrationTestSuite) StateSync() {
 		s.Require().NoError(err)
 		return stateSyncNodeHeight == runningNodeHeight
 	},
-		3*time.Minute,
-		500*time.Millisecond,
+		1*time.Minute,
+		time.Second,
 	)
 
 	// stop the state synching node.