From 3156a8252322e660f3509a3e67473f752359b96d Mon Sep 17 00:00:00 2001 From: Yuan Date: Thu, 6 Feb 2025 09:30:26 +0800 Subject: [PATCH] [GLUTEN-8574][VL]CI: adding Spark-344 unit tests on JDK8 and adding Spark-352 unit tests on JDK17 (#8591) adding Spark-344 unit tests on JDK8 and adding Spark-352 unit tests on JDK17 also removed the spark-344 source download step as we already installed in the docker image --- .github/workflows/velox_backend.yml | 132 ++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 9 deletions(-) diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 8735d74e0bad..cea185964597 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -759,8 +759,6 @@ jobs: path: /root/.m2/repository/org/apache/arrow/ - name: Prepare spark.test.home for Spark 3.4.4 (other tests) run: | - rm -rf /opt/shims/spark34 - bash .github/workflows/util/install_spark_resources.sh 3.4 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -780,7 +778,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-report-spark34 + name: test-report-spark34-jdk17 path: '**/surefire-reports/TEST-*.xml' - name: Upload golden files if: failure() @@ -789,6 +787,51 @@ jobs: name: golden-files-spark34 path: /tmp/tpch-approved-plan/** + run-spark-test-spark34-jdk8: + needs: build-native-lib-centos-7 + runs-on: ubuntu-20.04 + container: apache/gluten:centos-8 + steps: + - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v3 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v3 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare spark.test.home for Spark 3.4.4 (other tests) + run: | + dnf module -y install python39 && \ + alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ + pip3 install pyspark==3.4.4 cython && \ + pip3 install pandas pyarrow + - name: Build and Run unit test for Spark 3.4.4 (other tests) + run: | + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk + export SPARK_HOME=/opt/shims/spark34/spark_home/ + ls -l /opt/shims/spark34/spark_home/ + $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pceleborn -Pdelta -Phudi -Pspark-ut \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags \ + -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-report-spark34 + path: '**/surefire-reports/TEST-*.xml' + - name: Upload golden files + if: failure() + uses: actions/upload-artifact@v4 + with: + name: golden-files-spark34-jdk8 + path: /tmp/tpch-approved-plan/** run-spark-test-spark34-slow: needs: build-native-lib-centos-7 @@ -806,14 +849,9 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare - run: | - rm -rf /opt/shims/spark34 - bash .github/workflows/util/install_spark_resources.sh 3.4 - name: Build and Run unit test for Spark 3.4.4 (slow tests) run: | cd $GITHUB_WORKSPACE/ - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l /opt/shims/spark34/spark_home/ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -Phudi \ @@ -823,7 +861,39 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-report-spark34-slow + name: test-report-spark34-slow-jdk17 + path: '**/surefire-reports/TEST-*.xml' + + run-spark-test-spark34-slow-jdk8: + needs: build-native-lib-centos-7 + runs-on: ubuntu-20.04 + container: apache/gluten:centos-8 + steps: + - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v3 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v3 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Build and Run unit test for Spark 3.4.4 (slow tests) + run: | + cd $GITHUB_WORKSPACE/ + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk + export SPARK_HOME=/opt/shims/spark34/spark_home/ + ls -l /opt/shims/spark34/spark_home/ + $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pceleborn -Pdelta -Pspark-ut -Phudi \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ + -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-report-spark34-slow-jdk8 path: '**/surefire-reports/TEST-*.xml' run-spark-test-spark35: @@ -869,6 +939,50 @@ jobs: name: golden-files-spark35 path: /tmp/tpch-approved-plan/** + run-spark-test-spark35-jdk17: + needs: build-native-lib-centos-7 + runs-on: ubuntu-20.04 + container: apache/gluten:centos-8-jdk17 + steps: + - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v3 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v3 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare + run: | + dnf module -y install python39 && \ + alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ + pip3 install pyspark==3.5.2 cython && \ + pip3 install pandas pyarrow + - name: Build and Run unit test for Spark 3.5.2 (other tests) + run: | + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ ${EXTRA_FLAGS}" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-report-spark35-jdk17 + path: '**/surefire-reports/TEST-*.xml' + - name: Upload golden files + if: failure() + uses: actions/upload-artifact@v4 + with: + name: golden-files-spark35 + path: /tmp/tpch-approved-plan/** + run-spark-test-spark35-scala213: needs: build-native-lib-centos-7 runs-on: ubuntu-20.04