Merge pull request #781 from NVIDIA/branch-24.10

release 24.10 [skip ci]

.github/workflows/auto-merge.yml

@@ -18,25 +18,16 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-24.08
+      - branch-*
     types: [closed]
 
 jobs:
   auto-merge:
     if: github.event.pull_request.merged == true
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: branch-24.08 # force to fetch from latest upstream instead of PR ref
-
-      - name: auto-merge job
-        uses: ./.github/workflows/auto-merge
-        env:
-          OWNER: NVIDIA
-          REPO_NAME: spark-rapids-ml
-          HEAD: branch-24.08
-          BASE: branch-24.10
-          AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR
-
+    uses: NVIDIA/spark-rapids-common/.github/workflows/auto-merge.yml@main
+    with:
+      owner: ${{ github.repository_owner }}
+      repo: spark-rapids-ml
+      branch: ${{ github.event.pull_request.base.ref }}
+    secrets:
+      token: ${{ secrets.AUTOMERGE_TOKEN }}

.github/workflows/blossom-ci.yml

@@ -46,7 +46,8 @@ jobs:
         github.actor == 'GaryShen2008' ||
         github.actor == 'NvTimLiu' ||
         github.actor == 'YanxuanLiu' ||
-        github.actor == 'pxLi'
+        github.actor == 'pxLi' ||
+        github.actor == 'rishic3'
       )
     steps:
       - name: Check if comment is issued by authorized person

ci/Dockerfile

@@ -37,6 +37,6 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86
     && conda config --set solver libmamba
 
 # install cuML
-ARG CUML_VER=24.08
-RUN conda install -y -c rapidsai -c conda-forge -c nvidia cuml=$CUML_VER cuvs=$CUML_VER python=3.9 cuda-version=11.8 \
+ARG CUML_VER=24.10
+RUN conda install -y -c rapidsai -c conda-forge -c nvidia cuml=$CUML_VER cuvs=$CUML_VER python=3.10 cuda-version=11.8 numpy~=1.0 \
     && conda clean --all -f -y

docker/Dockerfile.pip

@@ -18,7 +18,7 @@ ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04
 
 ARG PYSPARK_VERSION=3.3.1
-ARG RAPIDS_VERSION=24.8.0
+ARG RAPIDS_VERSION=24.10.0
 ARG ARCH=amd64
 #ARG ARCH=arm64
 # Install packages to build spark-rapids-ml
@@ -41,6 +41,7 @@ RUN pip install --no-cache-dir \
     cudf-cu11~=${RAPIDS_VERSION} \
     cuml-cu11~=${RAPIDS_VERSION} \
     cuvs-cu11~=${RAPIDS_VERSION} \
+    numpy~=1.0 \
     --extra-index-url=https://pypi.nvidia.com
 
 # install python dependencies

docker/Dockerfile.python

@@ -17,7 +17,7 @@
 ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
 
-ARG CUML_VERSION=24.08
+ARG CUML_VERSION=24.10
 
 # Install packages to build spark-rapids-ml
 RUN apt update -y \
@@ -38,7 +38,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linu
 
 # install cuML
 
-RUN conda install -y -c rapidsai -c conda-forge -c nvidia python=3.9 cuda-version=11.8 cuml=$CUML_VERSION \
+RUN conda install -y -c rapidsai -c conda-forge -c nvidia python=3.10 cuda-version=11.8 cuml=$CUML_VERSION numpy~=1.0 \
     && conda clean --all -f -y
 
 # install python dependencies

docs/site/compatibility.md

@@ -28,7 +28,7 @@ Note: Spark does not provide a k-Nearest Neighbors (k-NN) implementation, but it
 
 | Spark Rapids ML | CUDA  | Spark  | Python |
 | :-------------- | :---- | :----- | :----- |
-| 1.0.0           | 11.5+ | 3.2.1+ | 3.9+   |
+| 1.0.0           | 11.4+ | 3.3+   | 3.10+  |
 
 
 ## Single vs Double precision inputs

docs/source/conf.py

@@ -9,7 +9,7 @@
 project = 'spark-rapids-ml'
 copyright = '2024, NVIDIA'
 author = 'NVIDIA'
-release = '24.08.0'
+release = '24.10.0'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

notebooks/aws-emr/README.md

@@ -41,7 +41,8 @@ If you already have a AWS EMR account, you can run the example notebooks on an E
 
   aws emr create-cluster \
   --name ${CLUSTER_NAME} \
-  --release-label emr-6.10.0 \
+  --release-label emr-7.3.0 \
+  --ebs-root-volume-size=32 \
   --applications Name=Hadoop Name=Livy Name=Spark Name=JupyterEnterpriseGateway \
   --service-role EMR_DefaultRole \
   --log-uri s3://${S3_BUCKET}/logs \

notebooks/aws-emr/init-bootstrap-action.sh

@@ -1,23 +1,28 @@
 #!/bin/bash
 
 set -ex
-
-sudo chmod a+rwx -R /sys/fs/cgroup/cpu,cpuacct
-sudo chmod a+rwx -R /sys/fs/cgroup/devices
 
-sudo yum install -y gcc openssl-devel bzip2-devel libffi-devel tar gzip wget make mysql-devel
-sudo bash -c "wget https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tgz && tar xzf Python-3.9.9.tgz && cd Python-3.9.9 && ./configure --enable-optimizations && make altinstall"
+sudo mkdir -p /spark-rapids-cgroup/devices
+sudo mount -t cgroup -o devices cgroupv1-devices /spark-rapids-cgroup/devices
+sudo chmod a+rwx -R /spark-rapids-cgroup
 
-RAPIDS_VERSION=24.8.0
+sudo yum update -y
+sudo yum install -y gcc bzip2-devel libffi-devel tar gzip wget make 
+sudo yum install -y mysql-devel --skip-broken
+sudo bash -c "wget https://www.python.org/ftp/python/3.10.9/Python-3.10.9.tgz && \
+tar xzf Python-3.10.9.tgz && cd Python-3.10.9 && \
+./configure --enable-optimizations && make altinstall"
+
+RAPIDS_VERSION=24.10.0
+
+sudo /usr/local/bin/pip3.10 install --upgrade pip
 
 # install scikit-learn 
-sudo /usr/local/bin/pip3.9 install scikit-learn
+sudo /usr/local/bin/pip3.10 install scikit-learn
 
 # install cudf and cuml
-sudo /usr/local/bin/pip3.9 install --no-cache-dir cudf-cu11==${RAPIDS_VERSION} \
-    cuml-cu11==${RAPIDS_VERSION} \
-    cuvs-cu11==${RAPIDS_VERSION} \
-    pylibraft-cu11==${RAPIDS_VERSION} \
-    rmm-cu11==${RAPIDS_VERSION} \
-    --extra-index-url=https://pypi.nvidia.com
+sudo /usr/local/bin/pip3.10 install --no-cache-dir cudf-cu12 --extra-index-url=https://pypi.nvidia.com --verbose
+sudo /usr/local/bin/pip3.10 install --no-cache-dir cuml-cu12 cuvs-cu12 --extra-index-url=https://pypi.nvidia.com --verbose
+
+sudo /usr/local/bin/pip3.10 list
 

notebooks/aws-emr/init-configurations.json

@@ -13,7 +13,7 @@
             "yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto",
             "yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables":"/usr/bin",
             "yarn.nodemanager.linux-container-executor.cgroups.mount":"true",
-            "yarn.nodemanager.linux-container-executor.cgroups.mount-path":"/sys/fs/cgroup",
+            "yarn.nodemanager.linux-container-executor.cgroups.mount-path":"/spark-rapids-cgroup",
             "yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"yarn",
             "yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"
         }
@@ -33,7 +33,7 @@
             {
                 "Classification":"cgroups",
                 "Properties":{
-                    "root":"/sys/fs/cgroup",
+                    "root":"/spark-rapids-cgroup",
                     "yarn-hierarchy":"yarn"
                 }
             }
@@ -68,14 +68,14 @@
             "spark.sql.execution.arrow.pyspark.enabled":"true",
             "spark.sql.execution.arrow.maxRecordsPerBatch":"100000",
             "spark.sql.cache.serializer":"com.nvidia.spark.ParquetCachedBatchSerializer",
-            "spark.pyspark.python":"python3.9",
-            "spark.pyspark.driver.python":"python3.9",
+            "spark.pyspark.python":"/usr/local/bin/python3.10",
+            "spark.pyspark.driver.python":"/usr/local/bin/python3.10",
             "spark.dynamicAllocation.enabled":"false",
             "spark.driver.memory":"20g",
             "spark.rpc.message.maxSize":"512",
             "spark.executorEnv.CUPY_CACHE_DIR":"/tmp/.cupy",
             "spark.executorEnv.NCCL_DEBUG":"INFO",
-            "spark.executorEnv.NCCL_SOCKET_IFNAME":"eth"
+            "spark.executorEnv.NCCL_SOCKET_IFNAME":"ens"
         }
     },
     {

notebooks/databricks/README.md

@@ -51,7 +51,7 @@ If you already have a Databricks account, you can run the example notebooks on a
       spark.task.resource.gpu.amount 1
       spark.databricks.delta.preview.enabled true
       spark.python.worker.reuse true
-      spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-24.06.1.jar:/databricks/spark/python
+      spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-24.08.1.jar:/databricks/spark/python
       spark.sql.execution.arrow.maxRecordsPerBatch 100000
       spark.rapids.memory.gpu.minAllocFraction 0.0001
       spark.plugins com.nvidia.spark.SQLPlugin

notebooks/databricks/init-pip-cuda-11.8.sh

@@ -4,8 +4,8 @@ SPARK_RAPIDS_ML_ZIP=/dbfs/path/to/zip/file
 # IMPORTANT: specify RAPIDS_VERSION fully 23.10.0 and not 23.10
 # also in general, RAPIDS_VERSION (python) fields should omit any leading 0 in month/minor field (i.e. 23.8.0 and not 23.08.0)
 # while SPARK_RAPIDS_VERSION (jar) should have leading 0 in month/minor (e.g. 23.08.2 and not 23.8.2)
-RAPIDS_VERSION=24.8.0
-SPARK_RAPIDS_VERSION=24.06.1
+RAPIDS_VERSION=24.10.0
+SPARK_RAPIDS_VERSION=24.08.1
 
 curl -L https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/${SPARK_RAPIDS_VERSION}/rapids-4-spark_2.12-${SPARK_RAPIDS_VERSION}-cuda11.jar -o /databricks/jars/rapids-4-spark_2.12-${SPARK_RAPIDS_VERSION}.jar