chore(ci): add back e2e workflows for dev profile to main branch (#6264)

* add workflows Signed-off-by: Bugen Zhao <[email protected]> * add parallel Signed-off-by: Bugen Zhao <[email protected]> * cleanup source e2e Signed-off-by: Bugen Zhao <[email protected]> Signed-off-by: Bugen Zhao <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
risingwavelabs · Nov 9, 2022 · b599ecc · b599ecc
1 parent 75d6459
commit b599ecc
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 22 deletions.
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
@@ -59,13 +59,3 @@ buildkite-agent artifact upload risedev-dev-"$profile"
 buildkite-agent artifact upload risingwave_regress_test-"$profile"
 buildkite-agent artifact upload ./sqlsmith-"$profile"
 buildkite-agent artifact upload ./compaction-test-"$profile"
-
-echo "--- upload misc"
-cp src/source/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
-buildkite-agent artifact upload ./avro-simple-schema.avsc
-
-cp src/source/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
-buildkite-agent artifact upload ./avro-complex-schema.avsc
-
-cp src/source/src/test_data/complex-schema ./proto-complex-schema
-buildkite-agent artifact upload ./proto-complex-schema
diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
@@ -28,10 +28,10 @@ buildkite-agent artifact download risedev-dev-"$profile" target/debug/
 mv target/debug/risingwave-"$profile" target/debug/risingwave
 mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
 
-echo "--- Download mise"
-buildkite-agent artifact download avro-simple-schema.avsc ./
-buildkite-agent artifact download avro-complex-schema.avsc ./
-buildkite-agent artifact download proto-complex-schema ./
+echo "--- Prepare data"
+cp src/source/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
+cp src/source/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
+cp src/source/src/test_data/complex-schema ./proto-complex-schema
 
 echo "--- Adjust permission"
 chmod +x ./target/debug/risingwave

diff --git a/ci/workflows/main.yml b/ci/workflows/main.yml
@@ -20,9 +20,21 @@ auto-retry: &auto-retry
       limit: 2
 
 steps:
+  - label: "build (dev mode)"
+    command: "ci/scripts/build.sh -t ci-dev -p ci-dev"
+    key: "build-dev"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "build (release mode)"
     command: "ci/scripts/build.sh -t ci-release -p ci-release"
-    key: "build"
+    key: "build-release"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
@@ -44,10 +56,31 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "end-to-end test (dev mode)"
+    command: "ci/scripts/e2e-test.sh -p ci-dev"
+    depends_on:
+      - "build-dev"
+      - "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - test-collector#v1.0.0:
+          files: "*-junit.xml"
+          format: "junit"
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "end-to-end test (release mode)"
     command: "ci/scripts/e2e-test.sh -p ci-release"
     depends_on:
-      - "build"
+      - "build-release"
       - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
@@ -65,10 +98,10 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (parallel) (release mode)"
-    command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
+  - label: "end-to-end test (parallel) (dev mode)"
+    command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
     depends_on:
-      - "build"
+      - "build-dev"
       - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
@@ -86,9 +119,11 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (parallel, in-memory) (release mode)"
-    command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release"
-    depends_on: "build"
+  - label: "end-to-end test (parallel) (release mode)"
+    command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
+    depends_on:
+      - "build-release"
+      - "docslt"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - seek-oss/aws-sm#v2.3.1:
@@ -105,6 +140,34 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "end-to-end test (parallel, in-memory) (release mode)"
+    command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release"
+    depends_on:
+      - "build-release"
+      - "docslt"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
+  - label: "end-to-end source test (release mode)"
+    command: "ci/scripts/e2e-source-test.sh -p ci-release"
+    depends_on: "build-release"
+    plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 5
+    retry: *auto-retry
+
   - label: "release"
     command: "ci/scripts/release.sh"
     if: build.tag != null