diff --git a/.github/actions/manylinux_2014_setup/action.yml b/.github/actions/manylinux_2014_setup/action.yml new file mode 100644 index 00000000..71bb3f0a --- /dev/null +++ b/.github/actions/manylinux_2014_setup/action.yml @@ -0,0 +1,111 @@ +name: "Setup manylinux2014 image" +description: "Installs/configures additional CI dependencies" +inputs: + aws-cli: + description: 'Setup aws-cli' + default: 0 + ninja-build: + description: 'Setup ninja-build' + default: 0 + vcpkg: + description: 'Setup vcpkg (installs to $GITHUB_WORKSPACE/vcpkg)' + default: 0 + openssl: + description: 'Setup OpenSSL (requires vcpkg to also be installed)' + default: 0 + ccache: + description: 'Setup Ccache' + default: 0 + jdk: + description: 'Setup JDK' + default: 0 + odbc: + description: 'Setup ODBC' + default: 0 + ssh: + description: 'Setup SSH' + default: 0 + glibc32: + description: 'Setup 32bit glibc' + default: 0 + nodejs: + description: 'Setup NodeJS' + default: 0 + gcc_4_8: + description: 'Setup GCC 4.8 (installs to /usr/bin/g++, default will still be GCC 10)' + default: 0 + python_alias: + description: 'Create an alias for python3 to python3.9' + default: 0 + +runs: + using: "composite" + steps: + - name: Setup general dependencies + shell: bash + run: scripts/setup_manylinux2014.sh general + + - name: Install AWS CLI + if: ${{ inputs.aws-cli == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh aws-cli + + - name: Setup dependencies for ODBC + if: ${{ inputs.odbc == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh odbc + + - name: Setup dependencies for ccache + if: ${{ inputs.ccache == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh ccache + + - name: Setup JDK + if: ${{ inputs.jdk == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh jdk + + - name: Setup SSH + if: ${{ inputs.ssh == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh ssh + + - name: Setup NodeJS + if: ${{ inputs.nodejs == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh nodejs + + - name: Setup 32bit compiler + if: ${{ inputs.glibc32 == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh glibc32 + + - name: Setup python3 as python3.9 + if: ${{ inputs.python_alias == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh python_alias + + - name: Setup old (GCC 4.8) compiler + if: ${{ inputs.gcc_4_8 == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh gcc_4_8 + + # Note instead of using scripts/setup_manylinux2014.sh vcpkg, we prefer to use + # lukka/run-vcpkg@v11.1 here as it configures vcpkg to cache to GH actions. + - name: Setup vcpkg + if: ${{ inputs.vcpkg == 1 }} + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + + - name: Install OpenSSL + if: ${{ inputs.openssl == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh openssl + + - name: Setup Ccache + if: ${{ inputs.ccache == 1 }} + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} \ No newline at end of file diff --git a/.github/actions/ubuntu_18_setup/action.yml b/.github/actions/ubuntu_18_setup/action.yml new file mode 100644 index 00000000..c01e8ce7 --- /dev/null +++ b/.github/actions/ubuntu_18_setup/action.yml @@ -0,0 +1,100 @@ +name: "Setup Ubuntu 18" +description: "Setup an Ubuntu 18 docker container with the required libraries" +inputs: + openssl: + description: 'OpenSSL' + default: 0 + python: + description: 'Python' + default: 1 + aarch64_cross_compile: + description: 'Install dependencies for aarch64 cross-compiling' + default: 0 + vcpkg: + description: 'Install vcpkg' + default: 0 + ccache: + description: 'Install ccache' + default: 0 + +runs: + using: "composite" + steps: + - name: Install + shell: bash + run: | + apt-get update -y -qq + apt-get install -y -qq software-properties-common + add-apt-repository ppa:git-core/ppa + apt-get update -y -qq + apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config + + - name: Install + shell: bash + if: ${{ inputs.aarch64_cross_compile == 1 }} + run: | + apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install Git 2.18.5 + shell: bash + run: | + wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz + tar xvf v2.18.5.tar.gz + cd git-2.18.5 + make + make prefix=/usr install + git --version + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install CMake 3.21 + shell: bash + run: | + wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh + chmod +x cmake-3.21.3-linux-x86_64.sh + ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local + cmake --version + + - name: Install Python 3.8 + if: ${{ inputs.python }} == 1 + shell: bash + run: | + wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz + tar xvf Python-3.8.17.tgz + cd Python-3.8.17 + mkdir -p pythonbin + ./configure --with-ensurepip=install + make -j + make install + python3.8 --version + python3.8 -m pip install pip + python3.8 -m pip install requests awscli + + - name: Version Check + shell: bash + run: | + ldd --version ldd + python3 --version + git --version + git log -1 --format=%h + + - name: Setup vcpkg + if: ${{ inputs.vcpkg == 1 }} + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + + - name: Setup Ccache + if: ${{ inputs.ccache == 1 }} + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} + + - name: Instal OpenSSL through vcpkg + if: ${{ inputs.openssl == 1 }} + shell: bash + run: | + cd $VCPKG_ROOT && ./vcpkg install openssl --triplet=${{ inputs.aarch64_cross_compile == 1 && 'arm64-linux' || 'x64-linux' }} \ No newline at end of file diff --git a/.github/workflows/ODBC.yml b/.github/workflows/ODBC.yml new file mode 100644 index 00000000..0a5e79e5 --- /dev/null +++ b/.github/workflows/ODBC.yml @@ -0,0 +1,279 @@ +name: JDBC +on: + push: + pull_request: + workflow_dispatch: + repository_dispatch: + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} + +jobs: + odbc-linux-amd64: + name: ODBC Linux (amd64) + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux2014_x86_64 + env: + GEN: ninja + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - uses: ./.github/actions/manylinux_2014_setup + with: + ninja-build: 1 + ccache: 1 + python_alias: 1 + aws-cli: 1 + odbc: 1 + + - name: Build + shell: bash + run: make release + + - name: ODBC Tests + shell: bash + if: ${{ inputs.skip_tests != 'true' }} + run: ./build/release/test/test_odbc + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + zip -j duckdb_odbc-linux-amd64.zip build/release/libduckdb_odbc.so linux_setup/unixodbc_setup.sh linux_setup/update_odbc_path.py + ./scripts/upload-assets-to-staging.sh github_release duckdb_odbc-linux-amd64.zip + - uses: actions/upload-artifact@v3 + with: + name: odbc-linux-amd64 + path: | + duckdb_odbc-linux-amd64.zip + + odbc-linux-aarch64: + name: ODBC Linux (aarch64) + runs-on: ubuntu-latest + container: + image: ubuntu:18.04 + needs: odbc-linux-amd64 + env: + GEN: ninja + DUCKDB_PLATFORM: linux_arm64 + ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - uses: ./.github/actions/ubuntu_18_setup + with: + ccache: 1 + aarch64_cross_compile: 1 + + - name: Install unixODBC + shell: bash + run: | # we need an x86 odbc_config tool to run cmake. fun. + apt-get remove -y unixodbc unixodbc-dev + CC=gcc ./scripts/install_unixodbc.sh + cp build/unixodbc/build/bin/odbc_config . + CC=aarch64-linux-gnu-gcc ./scripts/install_unixodbc.sh --host aarch64-linux-gnu + cp odbc_config build/unixodbc/build/bin/odbc_config + + - name: Build + shell: bash + run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make release + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + zip -j duckdb_odbc-linux-aarch64.zip build/release/libduckdb_odbc.so linux_setup/unixodbc_setup.sh linux_setup/update_odbc_path.py + ./scripts/upload-assets-to-staging.sh github_release duckdb_odbc-linux-aarch64.zip + + - uses: actions/upload-artifact@v3 + with: + name: odbc-linux-aarch64 + path: | + duckdb_odbc-linux-aarch64.zip + + + odbc-windows-amd64: + name: ODBC Windows (amd64) + runs-on: windows-latest + needs: odbc-linux-amd64 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} + + - name: Build + shell: bash + run: make release + + - name: ODBC Tests + if: ${{ inputs.skip_tests != 'true' }} + shell: bash + run: ./build/release/bin/Release/test_odbc.exe + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + choco install zip -y --force + zip -j duckdb_odbc-windows-amd64.zip ./build/release/bin/Release/duckdb_odbc.dll + ./scripts/upload-assets-to-staging.sh github_release duckdb_odbc-windows-amd64.zip + + - uses: actions/upload-artifact@v3 + with: + name: odbc-windows-amd64 + path: | + duckdb_odbc-windows-amd64.zip + + + odbc-osx-universal: + name: ODBC OSX (Universal) + runs-on: macos-14 + env: + GEN: ninja + ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config + OSX_BUILD_UNIVERSAL: 1 + needs: odbc-linux-amd64 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' }} + + - name: Install Ninja + run: brew install ninja + + - name: Install UnixODBC + shell: bash + run: CFLAGS="-arch x86_64 -arch arm64" ./scripts/install_unixodbc.sh + - name: Build + shell: bash + run: make release + - name: ODBC Tests + if: ${{ inputs.skip_tests != 'true' }} + shell: bash + run: ./build/release/test/test_odbc + - name: See if this actually universal + shell: bash + run: lipo -archs build/release/libduckdb_odbc.dylib | grep "x86_64 arm64" + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + zip -j duckdb_odbc-osx-universal.zip build/release/libduckdb_odbc.dylib + ./scripts/upload-assets-to-staging.sh github_release duckdb_odbc-osx-universal.zip + + - uses: actions/upload-artifact@v3 + with: + name: java-osx-universal + path: | + duckdb_odbc-osx-universal.zip + + debug: + name: ODBC Tests + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + runs-on: ubuntu-22.04 + needs: odbc-linux-amd64 + env: + BUILD_ODBC: 1 + GEN: ninja + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Dependencies + shell: bash + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq ninja-build unixodbc-dev + pip3 install pyodbc + + - name: Install nanodbc + shell: bash + run: | + wget https://github.com/nanodbc/nanodbc/archive/refs/tags/v2.14.0.tar.gz -O nanodbc.tgz + mkdir nanodbc + tar xvf nanodbc.tgz -C nanodbc --strip-components=1 + cd nanodbc + wget https://github.com/catchorg/Catch2/releases/download/v2.13.9/catch.hpp + cp catch.hpp test/catch/catch.hpp + sed -i -e "s/set(test_list/set(test_list odbc/" test/CMakeLists.txt + mkdir build + cd build + cmake -DNANODBC_DISABLE_TESTS=OFF .. + cmake --build . + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + + - name: Build + shell: bash + run: make debug + + - name: Test Standard ODBC tests + shell: bash + run: build/debug/test/test_odbc + + - name: Test Connection ODBC tests + shell: bash + run: | + chmod +x scripts/create_ini_file.sh + scripts/create_ini_file.sh + build/debug/test/test_connection_odbc + + - name: Test nanodbc + shell: bash + run: ./test/run_nanodbc_tests.sh + + - name: Test isql + shell: bash + run: ./test/run_isql_tests.sh + + - name: Test R ODBC + shell: bash + run: R -f test/rodbc.R + + - name: Test Python ODBC + shell: bash + run: ./test/run_pyodbc_tests.sh + diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bdded32..0a1e362e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.12.0) project(duckdb-odbc) find_package(ODBC) +find_package(Threads REQUIRED) set(CMAKE_CXX_STANDARD "11" CACHE STRING "C++ standard to enforce") set(CMAKE_VERBOSE_MAKEFILE OFF) @@ -23,14 +24,23 @@ if(ODBC_FOUND) if(WIN32) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) - add_subdirectory(winsetup) +# add_subdirectory(winsetup) list(APPEND LINK_LIB_LIST $<$:odbccp32>) list(APPEND LINK_LIB_LIST $<$:legacy_stdio_definitions>) endif() endif() -add_definitions(-DNO_FRAMEWORKS -DDUCKDB_EXTENSION_PARQUET_LINKED -DDUCKDB_EXTENSION_ICU_LINKED -DDUCKDB_EXTENSION_JSON_LINKED) +set(DUCKDB_SYSTEM_LIBS ${CMAKE_DL_LIBS}) +if(MSVC) + set(DUCKDB_SYSTEM_LIBS ${DUCKDB_SYSTEM_LIBS} ws2_32 rstrtmgr bcrypt) +endif() + +if(OSX_BUILD_UNIVERSAL) + SET(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "Build architectures for Mac OS X" FORCE) +endif() + +add_definitions(-DDUCKDB_BUILD_LIBRARY -DNO_FRAMEWORKS -DDUCKDB_EXTENSION_PARQUET_LINKED -DDUCKDB_EXTENSION_ICU_LINKED -DDUCKDB_EXTENSION_JSON_LINKED) include_directories(src/duckdb/src/include/ src/duckdb/src/include src/duckdb/third_party/concurrentqueue src/duckdb/third_party/fast_float src/duckdb/third_party/fastpforlib src/duckdb/third_party/fmt/include src/duckdb/third_party/fsst src/duckdb/third_party/httplib src/duckdb/third_party/hyperloglog src/duckdb/third_party/jaro_winkler src/duckdb/third_party/jaro_winkler/details src/duckdb/third_party/libpg_query src/duckdb/third_party/libpg_query/include src/duckdb/third_party/lz4 src/duckdb/third_party/mbedtls src/duckdb/third_party/mbedtls/include src/duckdb/third_party/mbedtls/library src/duckdb/third_party/miniz src/duckdb/third_party/pcg src/duckdb/third_party/re2 src/duckdb/third_party/skiplist src/duckdb/third_party/tdigest src/duckdb/third_party/utf8proc src/duckdb/third_party/utf8proc/include src/duckdb/extension/parquet/include src/duckdb/third_party/parquet src/duckdb/third_party/thrift src/duckdb/third_party/lz4 src/duckdb/third_party/snappy src/duckdb/third_party/zstd/include src/duckdb/third_party/mbedtls src/duckdb/third_party/mbedtls/include src/duckdb/extension/icu/include src/duckdb/extension/icu/third_party/icu/common src/duckdb/extension/icu/third_party/icu/i18n src/duckdb/extension/json/include src/duckdb/extension/json/yyjson/include ${ODBC_INCLUDE_DIRS} include) @@ -42,6 +52,7 @@ add_library(duckdb_odbc SHARED ${ALL_OBJECT_FILES} duckdb_odbc.def) set_target_properties(duckdb_odbc PROPERTIES DEFINE_SYMBOL "DUCKDB_ODBC_API") target_link_libraries(duckdb_odbc ${LINK_LIB_LIST} ) +target_link_libraries(duckdb_odbc Threads::Threads ${DUCKDB_SYSTEM_LIBS}) if(NOT CLANG_TIDY) add_subdirectory(test) diff --git a/CMakeLists.txt.in b/CMakeLists.txt.in index 19d2bc03..1287221f 100644 --- a/CMakeLists.txt.in +++ b/CMakeLists.txt.in @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.12.0) project(duckdb-odbc) find_package(ODBC) +find_package(Threads REQUIRED) set(CMAKE_CXX_STANDARD "11" CACHE STRING "C++ standard to enforce") set(CMAKE_VERBOSE_MAKEFILE OFF) @@ -23,14 +24,23 @@ if(ODBC_FOUND) if(WIN32) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) - add_subdirectory(winsetup) +# add_subdirectory(winsetup) list(APPEND LINK_LIB_LIST $<$:odbccp32>) list(APPEND LINK_LIB_LIST $<$:legacy_stdio_definitions>) endif() endif() -add_definitions(-DNO_FRAMEWORKS ${DEFINES}) +set(DUCKDB_SYSTEM_LIBS ${CMAKE_DL_LIBS}) +if(MSVC) + set(DUCKDB_SYSTEM_LIBS ${DUCKDB_SYSTEM_LIBS} ws2_32 rstrtmgr bcrypt) +endif() + +if(OSX_BUILD_UNIVERSAL) + SET(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "Build architectures for Mac OS X" FORCE) +endif() + +add_definitions(-DDUCKDB_BUILD_LIBRARY -DNO_FRAMEWORKS ${DEFINES}) include_directories(src/duckdb/src/include/ ${INCLUDE_FILES} ${ODBC_INCLUDE_DIRS} include) @@ -42,6 +52,7 @@ add_library(duckdb_odbc SHARED ${ALL_OBJECT_FILES} duckdb_odbc.def) set_target_properties(duckdb_odbc PROPERTIES DEFINE_SYMBOL "DUCKDB_ODBC_API") target_link_libraries(duckdb_odbc ${LINK_LIB_LIST} ${LIBRARY_FILES}) +target_link_libraries(duckdb_odbc Threads::Threads ${DUCKDB_SYSTEM_LIBS}) if(NOT CLANG_TIDY) add_subdirectory(test) diff --git a/Makefile b/Makefile index 53581f89..942b8db0 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,5 @@ -.PHONY: build test clean - -SEP= -JARS= - -ifeq ($(OS),Windows_NT) - # windows is weird - SEP=";" - JARS=build/release -else - SEP=":" - JARS=build/release -endif +.PHONY: release debug test clean -ARCH_OVERRIDE= -ifneq ($(OVERRIDE_JDBC_OS_ARCH),) - ARCH_OVERRIDE=-DOVERRIDE_JDBC_OS_ARCH=$(OVERRIDE_JDBC_OS_ARCH) -endif GENERATOR= @@ -24,16 +8,24 @@ ifeq ($(GEN),ninja) FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1 endif -JAR=$(JARS)/duckdb_jdbc.jar -TEST_JAR=$(JARS)/duckdb_jdbc_tests.jar -CP=$(JAR)$(SEP)$(TEST_JAR) +ODBC_CONFIG_FLAG= +ifneq ($(ODBC_CONFIG),) + ODBC_CONFIG_FLAG=-DODBC_CONFIG=${ODBC_CONFIG} +endif -test: - java -cp $(CP) org.duckdb.TestDuckDBJDBC +OSX_BUILD_UNIVERSAL_FLAG= +ifneq ($(OSX_BUILD_UNIVERSAL),) + OSX_BUILD_UNIVERSAL_FLAG=-DOSX_BUILD_UNIVERSAL=1 +endif release: mkdir -p build/release - cd build/release && cmake -DCMAKE_BUILD_TYPE=Release $(GENERATOR) $(ARCH_OVERRIDE) ../.. && cmake --build . --config Release + cd build/release && cmake -DCMAKE_BUILD_TYPE=Release $(GENERATOR) $(ODBC_CONFIG_FLAG) $(OSX_BUILD_UNIVERSAL_FLAG) ../.. && cmake --build . --config Release + +debug: + mkdir -p build/debug + cd build/debug && cmake -DCMAKE_BUILD_TYPE=Debug $(GENERATOR) $(ODBC_CONFIG_FLAG) $(OSX_BUILD_UNIVERSAL_FLAG) ../.. && cmake --build . --config Debug + clean: rm -rf build \ No newline at end of file diff --git a/README.md b/README.md index d23af75c..ad077892 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ BUILD_ODBC=1 DISABLE_SANITIZER=1 make debug -j The ODBC tests are written with the catch framework. To run the tests, run the following command from the main DuckDB repository: ```bash -build/debug/tools/odbc/test/test_odbc +build/debug/test/test_odbc ``` You can also individually run the tests by specifying the test name as an argument to the test executable: ```bash -build/debug/tools/odbc/test/test_odbc 'Test ALTER TABLE statement' +build/debug/test/test_odbc 'Test ALTER TABLE statement' ``` \ No newline at end of file diff --git a/scripts/install_unixodbc.sh b/scripts/install_unixodbc.sh new file mode 100755 index 00000000..61a8b890 --- /dev/null +++ b/scripts/install_unixodbc.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -x +rm -rf build/unixodbc +mkdir -p build/unixodbc +cd build/unixodbc +mkdir sources build +wget http://www.unixodbc.org/unixODBC-2.3.11.tar.gz || wget http://github.com/lurcher/unixODBC/releases/download/2.3.11/unixODBC-2.3.11.tar.gz +tar -xf unixODBC-2.3.11.tar.gz -C sources --strip-components 1 +cd sources +./configure --prefix `cd ../build; pwd` --disable-debug --disable-dependency-tracking --enable-static --enable-gui=no $@ +make -j install \ No newline at end of file diff --git a/scripts/setup_manylinux2014.sh b/scripts/setup_manylinux2014.sh new file mode 100755 index 00000000..8e9a7d70 --- /dev/null +++ b/scripts/setup_manylinux2014.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# This script is used to setup the additional dependencies required by duckdb +# for the manylinux2014 image. The reason for this to exist in a script is firstly to be able to +# easily reproduce the same environment locally. Secondly, these steps can be now used +# both in the x86 jobs as the aarch64 jobs on GH actions (aarch64 jobs need a special env). + +# Examples +# +### To install just basics: +# +# > scripts/setup_manylinux2014.sh general +# +### Install openssl (through vcpkg, which will be in /tmp/vcpkg), ccache and jdk +# +# > VCPKG_TARGET_DIR=/tmp scripts/setup_manylinux2014.sh general vcpkg openssl ccache jdk +# + +# Installs deps for a specific required dependency +install_deps() { + if [ "$1" = "general" ]; then + git config --global --add safe.directory '*' + yum install -y curl zip unzip tar + yum install -y ninja-build + + elif [ "$1" = "aws-cli" ]; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install + aws --version + + elif [ "$1" = "odbc" ]; then + yum install -y unixODBC-devel + + elif [ "$1" = "vcpkg" ]; then + # Note: it is preferred to use the lukka/run-vcpkg@v11 over this when running this + # in a Github Actions job. When running locally or in the aarch64, these can be used instead + # Also note that this action install. + # Note2: this installs vcpkg to $VCPKG_TARGET_DIR + ( + cd $VCPKG_TARGET_DIR ; + git clone https://github.com/Microsoft/vcpkg.git ; + git checkout a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 ; + cd vcpkg ; + ./bootstrap-vcpkg.sh + ) + export VCPKG_ROOT=$VCPKG_TARGET_DIR/vcpkg + + elif [ "$1" = "openssl" ]; then + yum install -y perl-IPC-Cmd + + elif [ "$1" = "ccache" ]; then + yum -y install ccache + + elif [ "$1" = "python_alias" ]; then + ln -fs /usr/local/bin/python3.9 /usr/local/bin/python3 + + elif [ "$1" = "jdk" ]; then + yum install -y java-11-openjdk-devel maven + + elif [ "$1" = "ssh" ]; then + yum install -y openssh-clients + + elif [ "$1" = "glibc32" ]; then + yum install -y libgcc*i686 libstdc++*i686 glibc*i686 libgfortran*i686 + + elif [ "$1" = "gcc_4_8" ]; then + yum install -y gcc-c++ + + elif [ "$1" = "nodejs" ]; then + yum install -y nodejs + + else + >&2 echo "unknown input for setup_manylinux2014.sh: '$1'" + exit $exit_code + fi +} + +for var in "$@" +do + install_deps $var +done + + diff --git a/scripts/upload-assets-to-staging.sh b/scripts/upload-assets-to-staging.sh new file mode 100755 index 00000000..ab3b9693 --- /dev/null +++ b/scripts/upload-assets-to-staging.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Main extension uploading script + +# Usage: ./scripts/upload-staging-asset.sh * +# : Folder to upload to +# : File to be uploaded + +if [ -z "$1" ] || [ -z "$2" ]; then + echo "Usage: ./scripts/upload-staging-asset.sh [... ]" + exit 1 +fi + +set -e + +# skip if repo is not in duckdb organization +if [ "$GITHUB_REPOSITORY_OWNER" != "duckdb" ]; then + echo "Repository is $GITHUB_REPOSITORY_OWNER (not duckdb)" + exit 0 +fi + +FOLDER="$1" +DRY_RUN_PARAM="" + +# dryrun if repo is not duckdb/duckdb-java +if [ "$GITHUB_REPOSITORY" != "duckdb/duckdb-java" ]; then + echo "Repository is $GITHUB_REPOSITORY (not duckdb/duckdb-java)" + DRY_RUN_PARAM="--dryrun" +fi +# dryrun if we are not in main +if [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo "git ref is $GITHUB_REF (not refs/heads/main)" + DRY_RUN_PARAM="--dryrun" +fi + +if [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + echo "overriding DRY_RUN_PARAM, forcing upload" + DRY_RUN_PARAM="" +fi + +# dryrun if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No access key available" + DRY_RUN_PARAM="--dryrun" +fi + + +TARGET=$(git log -1 --format=%h) + +if [ "$UPLOAD_ASSETS_TO_STAGING_TARGET" ]; then + TARGET="$UPLOAD_ASSETS_TO_STAGING_TARGET" +fi + +# decide target for staging +if [ "$OVERRIDE_GIT_DESCRIBE" ]; then + TARGET="$TARGET/$OVERRIDE_GIT_DESCRIBE" +fi + +python3 -m pip install awscli + +for var in "${@: 2}" +do + aws s3 cp $var s3://duckdb-staging/$TARGET/$GITHUB_REPOSITORY/$FOLDER/ $DRY_RUN_PARAM --region us-east-2 +done diff --git a/test/julia-test.jl b/test/julia-test.jl index d2aa6fe9..545fc0a3 100644 --- a/test/julia-test.jl +++ b/test/julia-test.jl @@ -11,7 +11,7 @@ ODBC.setunixODBC(;ODBCSYSINI=home, ODBCINSTINI=".odbcinst.ini", ODBCINI=string(h conn=ODBC.Connection("DSN=DuckDB;") duckdb_dir = pwd() -DBInterface.execute(conn,"CREATE TABLE test AS SELECT * FROM read_csv_auto('" * duckdb_dir * "/tools/odbc/test/test.csv')") +DBInterface.execute(conn,"CREATE TABLE test AS SELECT * FROM read_csv_auto('" * duckdb_dir * "test/test.csv')") df1 = DBInterface.execute(conn,"SELECT sum(a) as a, sum(b) as b, sum(c) as c FROM test")|>DataFrame df2 = DataFrame(a = Float64(3.3), b = Float64(36.0), c = Float64(1.0)); diff --git a/test/rodbc.R b/test/rodbc.R index 567cd9e1..68a62e01 100644 --- a/test/rodbc.R +++ b/test/rodbc.R @@ -4,7 +4,7 @@ install.packages(c("odbc", "DBI"), repos="https://cloud.r-project.org/", lib=tem library("DBI", lib.loc=tempdir()) library("odbc", lib.loc=tempdir()) -con <- dbConnect(odbc(), database=":memory:", driver=Sys.glob("build/debug/tools/odbc/libduckdb_odbc.*")) +con <- dbConnect(odbc(), database=":memory:", driver=Sys.glob("build/debug/libduckdb_odbc.*")) dbExecute(con, "CREATE TABLE fuu (i INTEGER, j STRING)") dbExecute(con, "INSERT INTO fuu VALUES (42, 'Hello'), (43, 'World'), (NULL, NULL)") diff --git a/test/run_isql_tests.sh b/test/run_isql_tests.sh index 6b0d2bbc..ed81187b 100755 --- a/test/run_isql_tests.sh +++ b/test/run_isql_tests.sh @@ -2,7 +2,7 @@ set -e -#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/tools/odbc/libduckdb_odbc.so" > ~/.odbcinst.ini +#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/libduckdb_odbc.so" > ~/.odbcinst.ini #echo -e "[DuckDB]\nDriver = DuckDB Driver\nDatabase=test.db\n" > ~/.odbc.ini BASE_DIR=$(dirname $0) @@ -12,17 +12,17 @@ if test -f test.db; then fi #Configuring ODBC files -$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -db test.db -D $(pwd)/build/debug/tools/odbc/libduckdb_odbc.so +$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -db test.db -D $(pwd)/build/debug/libduckdb_odbc.so export ASAN_OPTIONS=verify_asan_link_order=0 -python tools/odbc/test/isql-test.py isql +python test/isql-test.py isql if [[ $? != 0 ]]; then exit 1; fi # running isql with the option -e -rm test.db && python tools/odbc/test/isql-test.py isql -e +rm test.db && python test/isql-test.py isql -e if [[ $? != 0 ]]; then exit 1; fi diff --git a/test/run_nanodbc_tests.sh b/test/run_nanodbc_tests.sh index 879e59ee..1e5a9a71 100755 --- a/test/run_nanodbc_tests.sh +++ b/test/run_nanodbc_tests.sh @@ -2,7 +2,7 @@ set -e -#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/tools/odbc/libduckdb_odbc.so" > ~/.odbcinst.ini +#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/libduckdb_odbc.so" > ~/.odbcinst.ini #echo -e "[DuckDB]\nDriver = DuckDB Driver\nDatabase=:memory:\n" > ~/.odbc.ini BASE_DIR=$(dirname $0) @@ -22,7 +22,7 @@ case "$(uname -s)" in ;; esac -$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -D $(pwd)/build/debug/tools/odbc/libduckdb_odbc.${extension} +$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -D $(pwd)/build/debug/libduckdb_odbc.${extension} export NANODBC_TEST_CONNSTR_ODBC="DRIVER=DuckDB Driver;" export ASAN_OPTIONS=verify_asan_link_order=0 diff --git a/test/run_pyodbc_tests.sh b/test/run_pyodbc_tests.sh index 385fb5f5..75a00d81 100755 --- a/test/run_pyodbc_tests.sh +++ b/test/run_pyodbc_tests.sh @@ -2,17 +2,17 @@ set -e -#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/tools/odbc/libduckdb_odbc.so" > ~/.odbcinst.ini +#echo -e "[ODBC]\nTrace = yes\nTraceFile = /tmp/odbctrace\n\n[DuckDB Driver]\nDriver = "$(pwd)"/build/debug/libduckdb_odbc.so" > ~/.odbcinst.ini #echo -e "[DuckDB]\nDriver = DuckDB Driver\nDatabase=:memory:\n" > ~/.odbc.ini BASE_DIR=$(dirname $0) #Configuring ODBC files -$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -D $(pwd)/build/debug/tools/odbc/libduckdb_odbc.so +$BASE_DIR/../linux_setup/unixodbc_setup.sh -u -D $(pwd)/build/debug/libduckdb_odbc.so export ASAN_OPTIONS=verify_asan_link_order=0 -python3 tools/odbc/test/pyodbc-test.py +python3 test/pyodbc-test.py if [[ $? != 0 ]]; then exit 1; fi diff --git a/test/sql/storage_version/storage_version.db b/test/sql/storage_version/storage_version.db new file mode 100644 index 00000000..c816c3c2 Binary files /dev/null and b/test/sql/storage_version/storage_version.db differ diff --git a/test/tests/connect_with_ini.cpp b/test/tests/connect_with_ini.cpp index 8cb4e856..4d6bf95f 100644 --- a/test/tests/connect_with_ini.cpp +++ b/test/tests/connect_with_ini.cpp @@ -3,7 +3,7 @@ using namespace odbc_test; // If running this test on macOS you need to run the following script before executing the test -// tools/odbc/scripts/create_ini_file.sh +// scripts/create_ini_file.sh // Connect to the database using the ini file TEST_CASE("Test SQLConnect with Ini File", "[odbc]") {