e2e: separate testing strategies (normal vs scheduled) (osmosis-labs#…

…6303)

* separate e2e testing strategies

* run full test on sched

* move e2e back to test.yml

* change name to runScheduledTest

.github/workflows/e2e-full.yml

@@ -0,0 +1,96 @@
+name: E2E Full Daily Run
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  GO_VERSION: "1.20.5"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  get_diff:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Get git diff
+        uses: technote-space/[email protected]
+        with:
+          PATTERNS: |
+            **/**.wasm
+            **/**.go
+            **/**.mod
+            **/**.sum
+            Makefile
+            Dockerfile
+            .github/workflows/test.yml
+      - name: Set output
+        id: vars
+        run: echo "::set-output name=git_diff::$GIT_DIFF"
+    outputs:
+      git_diff: ${{ steps.vars.outputs.git_diff }}
+
+  e2e:
+    needs: get_diff
+    if: needs.get_diff.outputs.git_diff
+    runs-on: self-hosted
+    timeout-minutes: 20
+    steps:
+      - name: Clean up Pre-Existing E2E Docker containers
+        run: |
+          # Remove containers with names starting with "osmo-test-"
+          docker ps -aqf "name=osmo-test-*" | xargs -r docker rm -f
+
+          # Remove containers with names starting with "hermes-relayer"
+          docker ps -aqf "name=hermes-relayer*" | xargs -r docker rm -f
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build e2e image
+        uses: docker/build-push-action@v4
+        with:
+          load: true
+          context: .
+          tags: osmosis:debug
+          # Use experimental Cache backend API:
+          # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#cache-backend-api
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            BASE_IMG_TAG=debug
+      - name: Test e2e and Upgrade
+        run: make test-e2e-ci-scheduled
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: "./logs"
+      - name: Tar logs
+        if: failure()
+        run: tar cvzf ./logs.tgz ./logs
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs.tgz
+          path: ./logs.tgz
+        if: failure()
+      - name: 🧹 Clean up Osmosis Home
+        if: always()
+        run: rm -rf $HOME/.osmosisd/ || true
+      - name: Clean up E2E Docker containers
+        run: |
+          # Remove containers with names starting with "osmo-test-"
+          docker ps -aqf "name=osmo-test-*" | xargs -r docker rm -f
+
+          # Remove containers with names starting with "hermes-relayer"
+          docker ps -aqf "name=hermes-relayer*" | xargs -r docker rm -f

.github/workflows/test.yml

@@ -35,8 +35,11 @@ jobs:
             Makefile
             Dockerfile
             .github/workflows/test.yml
+      - name: Set output
+        id: vars
+        run: echo "::set-output name=git_diff::$GIT_DIFF"
     outputs:
-      git_diff: ${{ env.GIT_DIFF }}
+      git_diff: ${{ steps.vars.outputs.git_diff }}
 
   go-split-test-files:
     needs: get_diff

Makefile

@@ -372,12 +372,16 @@ test-sim-bench:
 # Utilizes Go cache.
 test-e2e: e2e-setup test-e2e-ci e2e-remove-resources
 
-# test-e2e-ci runs a full e2e test suite
+# test-e2e-ci runs a majority of e2e tests, only skipping the ones that are marked as scheduled tests
 # does not do any validation about the state of the Docker environment
 # As a result, avoid using this locally.
 test-e2e-ci:
 	@VERSION=$(VERSION) OSMOSIS_E2E=True OSMOSIS_E2E_DEBUG_LOG=False OSMOSIS_E2E_UPGRADE_VERSION=$(E2E_UPGRADE_VERSION) go test -mod=readonly -timeout=25m -v $(PACKAGES_E2E) -p 4
 
+# test-e2e-ci-scheduled runs every e2e test available, and is only run on a scheduled basis
+test-e2e-ci-scheduled:
+	@VERSION=$(VERSION) OSMOSIS_E2E_SCHEDULED=True OSMOSIS_E2E=True OSMOSIS_E2E_DEBUG_LOG=False OSMOSIS_E2E_UPGRADE_VERSION=$(E2E_UPGRADE_VERSION) go test -mod=readonly -timeout=25m -v $(PACKAGES_E2E) -p 4
+
 # test-e2e-debug runs a full e2e test suite but does
 # not attempt to delete Docker resources at the end.
 test-e2e-debug: e2e-setup

tests/e2e/e2e_cl_test.go

@@ -238,7 +238,7 @@ func (s *IntegrationTestSuite) ConcentratedLiquidity() {
 	var clwg sync.WaitGroup
 	var mu sync.Mutex
 
-	for i, _ := range positions {
+	for i := range positions {
 		clwg.Add(1)
 		go func(i int) { // Launch a goroutine
 			addr, positionSet := addresses[i], positions[i]

tests/e2e/e2e_setup_test.go

@@ -26,16 +26,19 @@ const (
 	skipCleanupEnv = "OSMOSIS_E2E_SKIP_CLEANUP"
 	// Environment variable name to determine what version we are upgrading to
 	upgradeVersionEnv = "OSMOSIS_E2E_UPGRADE_VERSION"
+	// Environment variable name to determine if we are running tests marked as scheduled
+	scheduledTestEv = "OSMOSIS_E2E_SCHEDULED"
 )
 
 type IntegrationTestSuite struct {
 	suite.Suite
 
-	configurer    configurer.Configurer
-	skipUpgrade   bool
-	skipIBC       bool
-	skipStateSync bool
-	mutex         sync.Mutex
+	configurer       configurer.Configurer
+	skipUpgrade      bool
+	skipIBC          bool
+	skipStateSync    bool
+	runScheduledTest bool
+	mutex            sync.Mutex
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
@@ -70,6 +73,14 @@ func (s *IntegrationTestSuite) SetupSuite() {
 	}
 	upgradeSettings.IsEnabled = !s.skipUpgrade
 
+	if str := os.Getenv(scheduledTestEv); len(str) > 0 {
+		s.runScheduledTest, err = strconv.ParseBool(str)
+		s.Require().NoError(err)
+		if s.runScheduledTest {
+			s.T().Logf("%s was true, running all tests", scheduledTestEv)
+		}
+	}
+
 	if str := os.Getenv(forkHeightEnv); len(str) > 0 {
 		upgradeSettings.ForkHeight, err = strconv.ParseInt(str, 0, 64)
 		s.Require().NoError(err)

tests/e2e/e2e_test.go

@@ -99,7 +99,7 @@ func (s *IntegrationTestSuite) TestAllE2E() {
 
 	// State Sync Dependent Tests
 
-	if s.skipStateSync {
+	if s.skipStateSync || !s.runScheduledTest {
 		s.T().Skip()
 	} else {
 		s.T().Run("StateSync", func(t *testing.T) {