From d57ff4ec347f46f5f3da5a91375d77d393a4c348 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 26 Apr 2023 09:46:37 +0200 Subject: [PATCH 1/5] add linux gcc4 --- .github/workflows/Linux.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index cc094043..5959067e 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -24,12 +24,14 @@ jobs: matrix: # Add commits/tags to build against other DuckDB versions duckdb_version: [ 'v0.7.1' ] - arch: ['linux_amd64', 'linux_arm64'] + arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] include: - arch: 'linux_amd64' - container: 'ubuntu:18.04' + container: 'ubuntu:16.04' - arch: 'linux_arm64' container: 'ubuntu:18.04' + - arch: 'linux_amd64_gcc4' + container: 'quay.io/pypa/manylinux2014_x86_64' env: GEN: ninja @@ -42,6 +44,7 @@ jobs: add-apt-repository ppa:git-core/ppa apt-get update -y -qq apt-get install -y -qq lsb-release sqlite3 ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client + - name: Install Git 2.18.5 if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} run: | @@ -51,6 +54,7 @@ jobs: make make prefix=/usr install git --version + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -61,12 +65,17 @@ jobs: run: | cd duckdb git checkout ${{ matrix.duckdb_version }} - + # Setup ccache - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ${{ github.job }}-${{ matrix.arch }} + + - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} + uses: ./duckdb/.github/actions/centos_7_setup + with: + openssl: 1 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} uses: ./duckdb/.github/actions/ubuntu_16_setup @@ -79,7 +88,7 @@ jobs: run: | export OPENSSL_ROOT_DIR=`pwd`/build/openssl/build echo "OPENSSL_ROOT_DIR=$OPENSSL_ROOT_DIR" >> $GITHUB_ENV - + # Build extension - name: Build extension env: @@ -90,10 +99,12 @@ jobs: run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" make release + - name: Build extension if: ${{ matrix.arch != 'linux_arm64'}} run: | make test_release + - uses: actions/upload-artifact@v2 with: name: ${{matrix.arch}}-extensions From 85bf27c45e4fc45df776bbcb8dfdaeff5f546204 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 26 Apr 2023 09:52:54 +0200 Subject: [PATCH 2/5] add upload step --- .github/workflows/Linux.yml | 25 ++++++++++++++++++++++++- .github/workflows/MacOS.yml | 28 +++++++++++++++++++++++++++- .github/workflows/Windows.yml | 24 +++++++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 5959067e..c2ec1dc3 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -109,4 +109,27 @@ jobs: with: name: ${{matrix.arch}}-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python3 -m pip install pip awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python3 -m pip install pip awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false + fi \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 745dc3b1..e7557ba4 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -89,4 +89,30 @@ jobs: with: name: macos_universal-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + echo $DUCKDB_VERSION + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + echo $DUCKDB_VERSION + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION osx_amd64 $BUCKET_NAME true + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION osx_arm64 $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION osx_amd64 $BUCKET_NAME false + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION osx_arm64 $BUCKET_NAME false + fi \ No newline at end of file diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index c57f3895..a48fd0a6 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -69,4 +69,26 @@ jobs: with: name: windows_x64-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false + fi \ No newline at end of file From 614bfe8cdb7883be914801384947fb3f402d904e Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 26 Apr 2023 09:57:51 +0200 Subject: [PATCH 3/5] dont build ccache on _gcc4 --- .github/workflows/Linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index c2ec1dc3..527f24b2 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -66,8 +66,9 @@ jobs: cd duckdb git checkout ${{ matrix.duckdb_version }} - # Setup ccache + # Setup ccache (not on _gcc4) - name: ccache + if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} uses: hendrikmuhs/ccache-action@v1.2 with: key: ${{ github.job }}-${{ matrix.arch }} From 0042e86441b4e654af4b490d2b2144feca7c2b93 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 26 Apr 2023 12:30:11 +0200 Subject: [PATCH 4/5] bump ubuntu 18 --- .github/workflows/Linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 527f24b2..5a31761d 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -27,7 +27,7 @@ jobs: arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] include: - arch: 'linux_amd64' - container: 'ubuntu:16.04' + container: 'ubuntu:18.04' - arch: 'linux_arm64' container: 'ubuntu:18.04' - arch: 'linux_amd64_gcc4' From 6055933ec67f2222f41b0c987696f0ccc76bd57d Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 26 Apr 2023 15:51:25 +0200 Subject: [PATCH 5/5] add scripts for upload --- scripts/extension-upload.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 scripts/extension-upload.sh diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh new file mode 100644 index 00000000..48fa20d9 --- /dev/null +++ b/scripts/extension-upload.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Usage: ./extension-upload.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") + +set -e + +ext="build/release/extension/$1/$1.duckdb_extension" + +# compress extension binary +gzip < $ext > "$1.duckdb_extension.gz" + +# upload compressed extension binary to S3 +aws s3 cp $1.duckdb_extension.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + +if [ $6 = 'true'] +then + aws s3 cp $1.duckdb_extension.gz s3://$5/$1/latest/$3/$4/$1.duckdb_extension.gz --acl public-read +fi +# also uplo \ No newline at end of file