From 96c668a83e523c8acd9a4b19035abedb24bf374b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 11 Jun 2024 10:28:39 +0200 Subject: [PATCH 01/12] android builds --- Makefile | 4 +++- deps/CMakeLists.txt | 7 +++++++ deps/patches/zlib.patch | 12 ++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 deps/patches/zlib.patch diff --git a/Makefile b/Makefile index 67ee613c..4e9fd826 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,8 @@ EXTENSION_FLAGS=\ #### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed) EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet;json" +DUCKDB_PLATFORM=android_armeabi-v7a + BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' ifeq (${BUILD_SHELL}, 0) BUILD_FLAGS += -DBUILD_SHELL=0 @@ -92,7 +94,7 @@ debug: release: mkdir -p build/release && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ + cmake -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=/opt/homebrew/share/android-ndk/build/cmake/android.toolchain.cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ cmake --build build/release --config Release ##### Client build diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 77e63709..c3e3675a 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -41,6 +41,7 @@ ExternalProject_Add( ZLIB URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/zlib1213.zip CONFIGURE_HANDLED_BY_BUILD TRUE + PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/zlib${PATCH_NAME_ENDING}.patch" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR} -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR} @@ -71,6 +72,7 @@ ExternalProject_Add( CONFIGURE_HANDLED_BY_BUILD TRUE CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR} + -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} -DCMAKE_POSITION_INDEPENDENT_CODE=ON @@ -142,6 +144,9 @@ ExternalProject_Add( -DBUILD_TESTING=OFF -DENABLE_CURL=OFF -DENABLE_TIFF=OFF + -DSQLITE3_VERSION=3.40.1 + -DSQLITE3_LIBRARY=${LOCAL_INSTALL_DIR}/lib/libsqlite3.a + -DSQLITE3_INCLUDE_DIR=${LOCAL_INSTALL_DIR}/include ) set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} PROJ) @@ -341,6 +346,8 @@ ExternalProject_Add( # Remove bindings -DBUILD_PYTHON_BINDINGS=OFF + + -DBUILD_WITHOUT_64BIT_OFFSET=1 ) diff --git a/deps/patches/zlib.patch b/deps/patches/zlib.patch new file mode 100644 index 00000000..30a20841 --- /dev/null +++ b/deps/patches/zlib.patch @@ -0,0 +1,12 @@ +diff --git a/zlib.map.bak b/zlib.map +index b330b60..9600ca1 100644 +--- a/zlib.map.bak ++++ b/zlib.map +@@ -15,7 +15,6 @@ ZLIB_1.2.0 { + zcfree; + z_errmsg; + gz_error; +- gz_intmax; + _*; + }; + From 1b578db9ae68c9b89bb4d02cf90604db9844b230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 18 Jun 2024 09:56:09 +0200 Subject: [PATCH 02/12] conditional config flag for gdal --- Makefile | 2 +- deps/CMakeLists.txt | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4e9fd826..c84a4283 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ debug: release: mkdir -p build/release && \ - cmake -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=/opt/homebrew/share/android-ndk/build/cmake/android.toolchain.cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ + cmake -DBUILD_UNITTESTS=0 -DBUILD_SHELL=0 -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=/opt/homebrew/share/android-ndk/build/cmake/android.toolchain.cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ cmake --build build/release --config Release ##### Client build diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index c3e3675a..199fe9f9 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -230,6 +230,10 @@ ExternalProject_Add( -DBUILD_DOCUMENTATION=OFF ) +if(${CMAKE_SIZEOF_VOID_P} LESS 8) + set(BUILD_WITHOUT_64BIT_OFFSET 1) +endif() + # GDAL ExternalProject_Add( GDAL @@ -347,7 +351,8 @@ ExternalProject_Add( # Remove bindings -DBUILD_PYTHON_BINDINGS=OFF - -DBUILD_WITHOUT_64BIT_OFFSET=1 + # are we building for 32 bit? + -DBUILD_WITHOUT_64BIT_OFFSET=${BUILD_WITHOUT_64BIT_OFFSET} ) From cbaec485025426728f4077ba71f926ab1b535d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 18 Jun 2024 09:57:32 +0200 Subject: [PATCH 03/12] restoring makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c84a4283..67ee613c 100644 --- a/Makefile +++ b/Makefile @@ -60,8 +60,6 @@ EXTENSION_FLAGS=\ #### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed) EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet;json" -DUCKDB_PLATFORM=android_armeabi-v7a - BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' ifeq (${BUILD_SHELL}, 0) BUILD_FLAGS += -DBUILD_SHELL=0 @@ -94,7 +92,7 @@ debug: release: mkdir -p build/release && \ - cmake -DBUILD_UNITTESTS=0 -DBUILD_SHELL=0 -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=/opt/homebrew/share/android-ndk/build/cmake/android.toolchain.cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ + cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ cmake --build build/release --config Release ##### Client build From ae94dd7a40ddff891e16d165830639397626161e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 18 Jun 2024 11:31:09 +0200 Subject: [PATCH 04/12] need to pass through ANDROID_ABI all the way --- CMakeLists.txt | 1 + deps/CMakeLists.txt | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95792382..785ff80d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ if(NOT EXISTS ${CMAKE_BINARY_DIR}/deps) -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${CMAKE_BINARY_DIR}/vcpkg_installed' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' + -DANDROID_ABI='${ANDROID_ABI}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -S ${CMAKE_CURRENT_SOURCE_DIR}/deps -B ${CMAKE_BINARY_DIR}/deps RESULT_VARIABLE DEPENDENCIES_GENERATE_RESULT) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index e962f3de..2e600609 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -51,6 +51,7 @@ ExternalProject_Add( -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -77,6 +78,7 @@ ExternalProject_Add( -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -133,6 +135,7 @@ ExternalProject_Add( -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ${CMAKE_PROJ_FLAGS} ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -164,6 +167,7 @@ ExternalProject_Add( -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -192,6 +196,7 @@ ExternalProject_Add( -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -219,6 +224,7 @@ ExternalProject_Add( -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' @@ -251,6 +257,7 @@ ExternalProject_Add( -DCMAKE_MODULE_PATH=${LOCAL_INSTALL_DIR}/lib/cmake -DCMAKE_FIND_ROOT_PATH=${LOCAL_INSTALL_DIR} ${CMAKE_FLAGS_POINTER_SIZE} + -DANDROID_ABI=${ANDROID_ABI} # vcpkg options -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' From c24b7c844a0ab886a49f90d3d9bc04977037acb4 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 5 Jul 2024 12:54:23 +0200 Subject: [PATCH 05/12] add wasm patch --- deps/patches/zlib_wasm.patch | 1 + 1 file changed, 1 insertion(+) create mode 100644 deps/patches/zlib_wasm.patch diff --git a/deps/patches/zlib_wasm.patch b/deps/patches/zlib_wasm.patch new file mode 100644 index 00000000..d3f5a12f --- /dev/null +++ b/deps/patches/zlib_wasm.patch @@ -0,0 +1 @@ + From 2dfaea1cbdcc4ed9d56d8cafd5bfb156df27ecb9 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 5 Jul 2024 12:58:51 +0200 Subject: [PATCH 06/12] update extension CI --- .../workflows/MainDistributionPipeline.yml | 2 +- .github/workflows/_extension_distribution.yml | 381 ------------------ 2 files changed, 1 insertion(+), 382 deletions(-) delete mode 100644 .github/workflows/_extension_distribution.yml diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 43eb3ff9..aaa76bc6 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -24,7 +24,7 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: ./.github/workflows/_extension_distribution.yml + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: duckdb_version: main extension_name: spatial diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml deleted file mode 100644 index 8bcf9ac3..00000000 --- a/.github/workflows/_extension_distribution.yml +++ /dev/null @@ -1,381 +0,0 @@ -# Reusable workflow for building DuckDB extensions using a standardized environment -# -# The workflow: -# - builds the extension using the CI workflow from the corresponding DuckDB version -# - uploads the extensions as gh actions artifacts in the following format: -# --extension- -# -# note: extensions are simply uploaded to GitHub actions, deploying the extensions is done a separate step. More info on -# this can be found in https://github.com/duckdb/extension-template - -name: Extension distribution -on: - workflow_call: - inputs: - # The name with which the extension will be built - extension_name: - required: true - type: string - # DuckDB version to build against, should in most cases be identical to - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default vcpkg commit used by this version of DuckDB - vcpkg_commit: - required: false - type: string - default: "a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6" - # Override the default script producing the matrices. Allows specifying custom matrices. - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - # Enable building the DuckDB Shell - build_duckdb_shell: - required: false - type: boolean - default: true - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - linux_matrix: ${{ steps.set-matrix-linux.outputs.linux_matrix }} - windows_matrix: ${{ steps.set-matrix-windows.outputs.windows_matrix }} - osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }} - wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os linux --output linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os osx --output osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os windows --output windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --select_os wasm --output wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - - - id: set-matrix-linux - run: | - linux_matrix="`cat linux_matrix.json`" - echo linux_matrix=$linux_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - - id: set-matrix-osx - run: | - osx_matrix="`cat osx_matrix.json`" - echo osx_matrix=$osx_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - - id: set-matrix-windows - run: | - windows_matrix="`cat windows_matrix.json`" - echo windows_matrix=$windows_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - - id: set-matrix-wasm - run: | - wasm_matrix="`cat wasm_matrix.json`" - echo wasm_matrix=$wasm_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - linux: - name: Linux - runs-on: ubuntu-latest - container: ${{ matrix.container }} - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.linux_matrix != '{}' && needs.generate_matrix.outputs.linux_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.linux_matrix)}} - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - GEN: Ninja - BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - - steps: - - name: Install required ubuntu packages - if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - - - name: Install Git 2.18.5 - if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - name: Setup ManyLinux2014 - if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }} - run: | - ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - continue-on-error: true - with: - key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - - name: Setup Ubuntu - if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} - uses: ./duckdb/.github/actions/ubuntu_18_setup - with: - aarch64_cross_compile: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} - - - name: Build extension - env: - GEN: ninja - CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} - CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - run: | - make release - - - name: Test extension - if: ${{ matrix.duckdb_arch != 'linux_arm64'}} - run: | - make test - - - uses: actions/upload-artifact@v3 - with: - if-no-files-found: error - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} - path: | - build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension - - macos: - name: MacOS - runs-on: macos-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.osx_matrix != '{}' && needs.generate_matrix.outputs.osx_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.osx_matrix)}} - env: - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }} - GEN: Ninja - BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Install Ninja - run: | - brew install ninja - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - continue-on-error: true - with: - key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} - - - name: Build extension - shell: bash - env: - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - run: | - make release - - - name: Test Extension - if: ${{ matrix.osx_build_arch == 'x86_64'}} - shell: bash - run: | - make test - - - uses: actions/upload-artifact@v3 - with: - if-no-files-found: error - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} - path: | - build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension - - windows: - name: Windows - runs-on: windows-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.windows_matrix != '{}' && needs.generate_matrix.outputs.windows_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.windows_matrix)}} - env: - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - CC: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 'gcc' || '' }} - CXX: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 'g++' || '' }} - - steps: - - name: Keep \n line endings - shell: bash - run: | - git config --global core.autocrlf false - git config --global core.eol lf - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - uses: r-lib/actions/setup-r@v2 - if: matrix.duckdb_arch == 'windows_amd64_rtools' - with: - r-version: 'devel' - update-rtools: true - rtools-version: '42' # linker bug in 43 - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - continue-on-error: true - with: - key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} - - - name: Fix for MSVC issue - shell: bash - env: - OVERLAY_TRIPLET_SRC: ${{ github.workspace }}/vcpkg/triplets/community/x64-windows-static-md.cmake - OVERLAY_TRIPLET_DST: ${{ github.workspace }}/overlay_triplets/x64-windows-static-md.cmake - run: | - mkdir overlay_triplets - cp $OVERLAY_TRIPLET_SRC $OVERLAY_TRIPLET_DST - echo "set(VCPKG_PLATFORM_TOOLSET_VERSION "14.39")" >> $OVERLAY_TRIPLET_DST - - - name: Build & test extension - env: - VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/overlay_triplets" - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} - run: | - make test_release - - - uses: actions/upload-artifact@v3 - with: - if-no-files-found: error - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} - path: | - build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension - - wasm: - name: DuckDB-Wasm - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.wasm_matrix != '{}' && needs.generate_matrix.outputs.wasm_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.wasm_matrix)}} - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - GEN: Ninja - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: mymindstorm/setup-emsdk@v13 - with: - version: 'latest' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - continue-on-error: true - with: - key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - - name: Build Wasm module - run: | - make ${{ matrix.duckdb_arch }} - - - uses: actions/upload-artifact@v3 - with: - if-no-files-found: error - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} - path: | - build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm From 800d3177d5658be84d8aca5b0e2bb5afee6c8204 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 5 Jul 2024 13:20:13 +0200 Subject: [PATCH 07/12] fix wasm patch.. maybe? --- deps/patches/zlib_wasm.patch | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/patches/zlib_wasm.patch b/deps/patches/zlib_wasm.patch index d3f5a12f..e69de29b 100644 --- a/deps/patches/zlib_wasm.patch +++ b/deps/patches/zlib_wasm.patch @@ -1 +0,0 @@ - From f3271d3ac3e756053a716277c66357c5a28349eb Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 5 Jul 2024 13:25:05 +0200 Subject: [PATCH 08/12] remove carriage returns --- deps/patches/zlib.patch | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deps/patches/zlib.patch b/deps/patches/zlib.patch index 30a20841..7de407bd 100644 --- a/deps/patches/zlib.patch +++ b/deps/patches/zlib.patch @@ -3,10 +3,10 @@ index b330b60..9600ca1 100644 --- a/zlib.map.bak +++ b/zlib.map @@ -15,7 +15,6 @@ ZLIB_1.2.0 { - zcfree; - z_errmsg; - gz_error; -- gz_intmax; - _*; - }; - + zcfree; + z_errmsg; + gz_error; +- gz_intmax; + _*; + }; + From 976c1ee643b6d456d4f4605f61c62edfe67fc562 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 2 Aug 2024 10:58:16 +0200 Subject: [PATCH 09/12] bump --- deps/patches/zlib.patch | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/patches/zlib.patch b/deps/patches/zlib.patch index 7de407bd..ca782c53 100644 --- a/deps/patches/zlib.patch +++ b/deps/patches/zlib.patch @@ -9,4 +9,3 @@ index b330b60..9600ca1 100644 - gz_intmax; _*; }; - From 51b4b1402b0f6337fa905a701ca7335c62238fb2 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 2 Aug 2024 11:10:34 +0200 Subject: [PATCH 10/12] try use zlib from vcpkg instead --- deps/CMakeLists.txt | 52 ++++++++++++++++++++++++--------------------- vcpkg.json | 3 ++- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 2e600609..b362fc9d 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -37,38 +37,37 @@ if (EMSCRIPTEN) endif() # ZLIB -ExternalProject_Add( - ZLIB - URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/zlib1213.zip - CONFIGURE_HANDLED_BY_BUILD TRUE - PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/zlib${PATCH_NAME_ENDING}.patch" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR} - -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR} - -DCMAKE_MODULE_PATH=${LOCAL_INSTALL_DIR}/lib/cmake - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_SHARED_LIBS=OFF - ${CMAKE_FLAGS_POINTER_SIZE} - -DANDROID_ABI=${ANDROID_ABI} +#ExternalProject_Add( +# ZLIB +# URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/zlib1213.zip +# CONFIGURE_HANDLED_BY_BUILD TRUE +# PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/zlib${PATCH_NAME_ENDING}.patch" +# CMAKE_ARGS +# -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR} +# -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR} +# -DCMAKE_MODULE_PATH=${LOCAL_INSTALL_DIR}/lib/cmake +# -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +# -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} +# -DCMAKE_POSITION_INDEPENDENT_CODE=ON +# -DBUILD_SHARED_LIBS=OFF +# ${CMAKE_FLAGS_POINTER_SIZE} +# -DANDROID_ABI=${ANDROID_ABI} # vcpkg options - -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' - -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' - -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' - -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -) +# -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' +# -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' +# -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' +# -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' +#) # Use the system sqlite binary on Emscripten, iOS and Android if (EMSCRIPTEN OR IOS OR ANDROID) find_program(EXE_SQLITE3 sqlite3) endif() -set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} ZLIB) +set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES}) # SQLite3 ExternalProject_Add( SQLITE3 - DEPENDS ZLIB URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/sqlite3 CONFIGURE_HANDLED_BY_BUILD TRUE CMAKE_ARGS @@ -84,6 +83,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DZLIB_USE_STATIC_LIBS=ON # Propagate to FindZLIB.cmake ) set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} SQLITE3) @@ -91,7 +91,6 @@ if(SPATIAL_USE_NETWORK) # CURL ExternalProject_Add( CURL - DEPENDS ZLIB URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/curl-7.87.0.zip CONFIGURE_HANDLED_BY_BUILD TRUE CMAKE_ARGS @@ -114,6 +113,7 @@ ExternalProject_Add( -DCURL_USE_LIBSSH=OFF -DOPENSSL_USE_STATIC_LIBS=ON # Propagate to FindOpenSSL.cmake -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} + -DZLIB_USE_STATIC_LIBS=ON # Propagate to FindZLIB.cmake ) set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} CURL) endif() @@ -122,7 +122,7 @@ endif() # PROJ ExternalProject_Add( PROJ - DEPENDS SQLITE3 ZLIB + DEPENDS SQLITE3 URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/proj-9.1.1.zip CONFIGURE_HANDLED_BY_BUILD TRUE PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/proj${PATCH_NAME_ENDING}.patch" @@ -150,6 +150,7 @@ ExternalProject_Add( -DSQLITE3_VERSION=3.40.1 -DSQLITE3_LIBRARY=${LOCAL_INSTALL_DIR}/lib/libsqlite3.a -DSQLITE3_INCLUDE_DIR=${LOCAL_INSTALL_DIR}/include + -DZLIB_USE_STATIC_LIBS=ON # Propagate to FindZLIB.cmake ) set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} PROJ) @@ -359,6 +360,9 @@ ExternalProject_Add( # Remove bindings -DBUILD_PYTHON_BINDINGS=OFF + # Zlib + -DZLIB_USE_STATIC_LIBS=ON # Propagate to FindZLIB.cmake + # are we building for 32 bit? -DBUILD_WITHOUT_64BIT_OFFSET=${BUILD_WITHOUT_64BIT_OFFSET} ) diff --git a/vcpkg.json b/vcpkg.json index 0fa0cf26..415b4fe1 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,5 +1,6 @@ { "dependencies": [ - "openssl" + "openssl", + "zlib" ] } From 8c9dd58794340335c30d590958d96d413f777950 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 2 Aug 2024 14:01:05 +0200 Subject: [PATCH 11/12] fix new context state manager --- spatial/src/spatial/gdal/file_handler.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spatial/src/spatial/gdal/file_handler.cpp b/spatial/src/spatial/gdal/file_handler.cpp index cb50144d..2e80598a 100644 --- a/spatial/src/spatial/gdal/file_handler.cpp +++ b/spatial/src/spatial/gdal/file_handler.cpp @@ -385,10 +385,9 @@ string GDALClientContextState::GetPrefix(const string &value) const { } GDALClientContextState &GDALClientContextState::GetOrCreate(ClientContext &context) { - if (!context.registered_state["gdal"]) { - context.registered_state["gdal"] = make_uniq(context); - } - return *dynamic_cast(context.registered_state["gdal"].get()); + auto &manager = *context.registered_state; + auto state = manager.GetOrCreate("gdal", context); + return state->Cast(); } } // namespace gdal From ca4825fa2ae03fa28321a998d190bdf1b6be2ad2 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Mon, 5 Aug 2024 00:12:07 +0200 Subject: [PATCH 12/12] disable building shell --- .github/workflows/MainDistributionPipeline.yml | 1 + Makefile | 2 +- deps/patches/zlib.patch | 11 ----------- deps/patches/zlib_wasm.patch | 0 4 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 deps/patches/zlib.patch delete mode 100644 deps/patches/zlib_wasm.patch diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index aaa76bc6..d5e9b1dc 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -28,6 +28,7 @@ jobs: with: duckdb_version: main extension_name: spatial + build_duckdb_shell: false # Dont build the shell, doesnt work on windows due to sqlite lib conflicts duckdb-stable-deploy: name: Deploy extension binaries diff --git a/Makefile b/Makefile index 67ee613c..eac88fa2 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ clean: # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(VCPKG_EMSDK_FLAGS) +WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" ${TOOLCHAIN_FLAGS} $(VCPKG_EMSDK_FLAGS) WASM_CXX_MVP_FLAGS= WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread diff --git a/deps/patches/zlib.patch b/deps/patches/zlib.patch deleted file mode 100644 index ca782c53..00000000 --- a/deps/patches/zlib.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/zlib.map.bak b/zlib.map -index b330b60..9600ca1 100644 ---- a/zlib.map.bak -+++ b/zlib.map -@@ -15,7 +15,6 @@ ZLIB_1.2.0 { - zcfree; - z_errmsg; - gz_error; -- gz_intmax; - _*; - }; diff --git a/deps/patches/zlib_wasm.patch b/deps/patches/zlib_wasm.patch deleted file mode 100644 index e69de29b..00000000