Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/branch-25.02' into raza-10661
Browse files Browse the repository at this point in the history
  • Loading branch information
gerashegalov committed Jan 10, 2025
2 parents a8762e2 + 139dd58 commit 6fc0c81
Show file tree
Hide file tree
Showing 339 changed files with 9,230 additions and 11,287 deletions.
14 changes: 8 additions & 6 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2020-2024, NVIDIA CORPORATION.
# Copyright (c) 2020-2025, NVIDIA CORPORATION.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand All @@ -9,8 +9,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.


/jenkins/ @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @gerashegalov
pom.xml @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @gerashegalov
/dist/ @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @gerashegalov
/.github/ @jlowe @revans2 @tgravescs @GaryShen2008 @NvTimLiu @gerashegalov
# Build-related
/jenkins/ @NVIDIA/sparkrapids-cicd-codeowners
pom.xml @NVIDIA/sparkrapids-cicd-codeowners
/dist/ @NVIDIA/sparkrapids-cicd-codeowners
/.github/ @NVIDIA/sparkrapids-cicd-codeowners
/build/ @NVIDIA/sparkrapids-cicd-codeowners
/scripts/ @NVIDIA/sparkrapids-cicd-codeowners
3 changes: 2 additions & 1 deletion .github/workflows/blossom-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ jobs:
github.actor == 'Feng-Jiang28' ||
github.actor == 'SurajAralihalli' ||
github.actor == 'jihoonson' ||
github.actor == 'ustcfy'
github.actor == 'ustcfy' ||
github.actor == 'knoguchi22'
)
steps:
- name: Check if comment is issued by authorized person
Expand Down
58 changes: 58 additions & 0 deletions .github/workflows/license-header-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright (c) 2024, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A workflow to check copyright/license header
name: license header check

on:
pull_request:
types: [opened, synchronize, reopened]

jobs:
license-header-check:
runs-on: ubuntu-latest
if: "!contains(github.event.pull_request.title, '[bot]')"
steps:
- name: Get checkout depth
run: |
echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 10 ))" >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: ${{ env.PR_FETCH_DEPTH }}

- name: license-header-check
uses: NVIDIA/spark-rapids-common/license-header-check@main
with:
included_file_patterns: |
*.yml,
*.yaml,
*.sh,
*.xml,
*.properties,
*.scala,
*.py,
build/*,
*.cpp,
*Dockerfile*,
*Jenkinsfile*,
*.ini,
*.java,
*.fbs
excluded_file_patterns: |
*target/*,
thirdparty/*,
sql-plugin/src/main/java/com/nvidia/spark/rapids/format/*
17 changes: 15 additions & 2 deletions .github/workflows/mvn-verify-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,20 @@ jobs:
id: generateCacheKey
run: |
set -x
cacheKey="${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')"
depsSHA1=$(. .github/workflows/mvn-verify-check/get-deps-sha1.sh 2.12)
cacheKey="${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-${depsSHA1}"
echo "dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT
- name: Cache local Maven repository
id: cache
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ env.dailyCacheKey }}
restore-keys: ${{ runner.os }}-maven-
- name: populate-daily-cache
timeout-minutes: 30
continue-on-error: true
if: steps.cache.outputs.cache-hit != 'true'
env:
SCALA_VER: '2.12'
Expand Down Expand Up @@ -117,6 +121,7 @@ jobs:
java-version: 8

- name: Cache local Maven repository
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
Expand Down Expand Up @@ -165,16 +170,20 @@ jobs:
id: generateCacheKey
run: |
set -x
cacheKey="${{ runner.os }}-maven-scala213-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')"
depsSHA1=$(. .github/workflows/mvn-verify-check/get-deps-sha1.sh 2.13)
cacheKey="${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-${depsSHA1}"
echo "scala213dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT
- name: Cache local Maven repository
id: cache
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ env.scala213dailyCacheKey }}
restore-keys: ${{ runner.os }}-maven-
- name: populate-daily-cache
timeout-minutes: 30
continue-on-error: true
if: steps.cache.outputs.cache-hit != 'true'
env:
SCALA_VER: '2.13'
Expand Down Expand Up @@ -221,6 +230,7 @@ jobs:
java-version: 17

- name: Cache local Maven repository
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
Expand Down Expand Up @@ -276,6 +286,7 @@ jobs:
java-version: 17

- name: Cache local Maven repository
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
Expand Down Expand Up @@ -330,6 +341,7 @@ jobs:
java-version: ${{ matrix.java-version }}

- name: Cache local Maven repository
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
Expand Down Expand Up @@ -377,6 +389,7 @@ jobs:
java-version: 11

- name: Cache local Maven repository
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.m2
Expand Down
36 changes: 36 additions & 0 deletions .github/workflows/mvn-verify-check/get-deps-sha1.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash

# Copyright (c) 2024, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

scala_ver=${1:-"2.12"}
base_URL="https://oss.sonatype.org/service/local/artifact/maven/resolve"
project_jni="spark-rapids-jni"
project_private="rapids-4-spark-private_${scala_ver}"

jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-jni.version -DforceStdout)
private_ver=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-private.version -DforceStdout)

jni_sha1=$(curl -s -H "Accept: application/json" \
"${base_URL}?r=snapshots&g=com.nvidia&a=${project_jni}&v=${jni_ver}&c=&e=jar&wt=json" \
| jq .data.sha1) || $(date +'%Y-%m-%d')
private_sha1=$(curl -s -H "Accept: application/json" \
"${base_URL}?r=snapshots&g=com.nvidia&a=${project_private}&v=${private_ver}&c=&e=jar&wt=json" \
| jq .data.sha1) || $(date +'%Y-%m-%d')

sha1md5=$(echo -n "${jni_sha1}_${private_sha1}" | md5sum | awk '{print $1}')

echo $sha1md5
21 changes: 12 additions & 9 deletions .github/workflows/mvn-verify-check/populate-daily-cache.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,27 +14,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -x
max_retry=3; delay=30; i=1
set -e
set -o pipefail

if [[ $SCALA_VER == '2.12' ]]; then
pom='pom.xml'
elif [[ $SCALA_VER == '2.13' ]]; then
pom='scala2.13/pom.xml'
fi

max_retry=3; delay=30; i=1
while true; do
buildvers=($(python build/get_buildvers.py no_snapshots $pom | tr -d ',')) &&
{
python build/get_buildvers.py "no_snapshots.buildvers" $pom | tr -d ',' | \
xargs -n 1 -I {} bash -c \
"mvn $COMMON_MVN_FLAGS --file $pom -Dbuildver={} de.qaware.maven:go-offline-maven-plugin:resolve-dependencies"

for buildver in "${buildvers[@]}"; do
mvn $COMMON_MVN_FLAGS --file $pom -Dbuildver=$buildver de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
done
} && {
# compile base versions to cache scala compiler and compiler bridge
mvn $COMMON_MVN_FLAGS --file $pom \
process-test-resources -pl sql-plugin-api -am
mvn $COMMON_MVN_FLAGS --file $pom process-test-resources -pl sql-plugin-api -am
} && break || {
if [[ $i -le $max_retry ]]; then
echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
else
echo "mvn command failed. Exit 1"; exit 1
fi
}
done
done
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,5 @@ scalastyle-output.xml
scalastyle.txt
target/
cufile.log
cudf_log.txt
build/*.class
Loading

0 comments on commit 6fc0c81

Please sign in to comment.