diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index cc094043..5a31761d 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -24,12 +24,14 @@ jobs: matrix: # Add commits/tags to build against other DuckDB versions duckdb_version: [ 'v0.7.1' ] - arch: ['linux_amd64', 'linux_arm64'] + arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] include: - arch: 'linux_amd64' container: 'ubuntu:18.04' - arch: 'linux_arm64' container: 'ubuntu:18.04' + - arch: 'linux_amd64_gcc4' + container: 'quay.io/pypa/manylinux2014_x86_64' env: GEN: ninja @@ -42,6 +44,7 @@ jobs: add-apt-repository ppa:git-core/ppa apt-get update -y -qq apt-get install -y -qq lsb-release sqlite3 ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client + - name: Install Git 2.18.5 if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} run: | @@ -51,6 +54,7 @@ jobs: make make prefix=/usr install git --version + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -61,12 +65,18 @@ jobs: run: | cd duckdb git checkout ${{ matrix.duckdb_version }} - - # Setup ccache + + # Setup ccache (not on _gcc4) - name: ccache + if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} uses: hendrikmuhs/ccache-action@v1.2 with: key: ${{ github.job }}-${{ matrix.arch }} + + - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} + uses: ./duckdb/.github/actions/centos_7_setup + with: + openssl: 1 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} uses: ./duckdb/.github/actions/ubuntu_16_setup @@ -79,7 +89,7 @@ jobs: run: | export OPENSSL_ROOT_DIR=`pwd`/build/openssl/build echo "OPENSSL_ROOT_DIR=$OPENSSL_ROOT_DIR" >> $GITHUB_ENV - + # Build extension - name: Build extension env: @@ -90,12 +100,37 @@ jobs: run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" make release + - name: Build extension if: ${{ matrix.arch != 'linux_arm64'}} run: | make test_release + - uses: actions/upload-artifact@v2 with: name: ${{matrix.arch}}-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python3 -m pip install pip awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python3 -m pip install pip awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false + fi \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 745dc3b1..e7557ba4 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -89,4 +89,30 @@ jobs: with: name: macos_universal-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + echo $DUCKDB_VERSION + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + echo $DUCKDB_VERSION + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION osx_amd64 $BUCKET_NAME true + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION osx_arm64 $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION osx_amd64 $BUCKET_NAME false + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION osx_arm64 $BUCKET_NAME false + fi \ No newline at end of file diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index c57f3895..a48fd0a6 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -69,4 +69,26 @@ jobs: with: name: windows_x64-extensions path: | - build/release/extension/spatial/spatial.duckdb_extension \ No newline at end of file + build/release/extension/spatial/spatial.duckdb_extension + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + run: | + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + python -m pip install awscli + ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false + fi \ No newline at end of file diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh new file mode 100644 index 00000000..48fa20d9 --- /dev/null +++ b/scripts/extension-upload.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Usage: ./extension-upload.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") + +set -e + +ext="build/release/extension/$1/$1.duckdb_extension" + +# compress extension binary +gzip < $ext > "$1.duckdb_extension.gz" + +# upload compressed extension binary to S3 +aws s3 cp $1.duckdb_extension.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + +if [ $6 = 'true'] +then + aws s3 cp $1.duckdb_extension.gz s3://$5/$1/latest/$3/$4/$1.duckdb_extension.gz --acl public-read +fi +# also uplo \ No newline at end of file