Nightly builds on linux #32
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly builds on linux | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| schedule: | |
| - cron: "0 2 * * *" # Every night at 2 AM UTC (9 PM EST; 10 PM EDT) | |
| workflow_dispatch: | |
| inputs: | |
| LIBTILEDB_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB | |
| default: dev | |
| TILEDB_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Py | |
| default: dev | |
| TILEDB_R_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-R | |
| default: master | |
| jobs: | |
| libtiledb: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone TileDB | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB | |
| ref: ${{ github.event.inputs.LIBTILEDB_REF || 'dev' }} | |
| - name: Download release tarball | |
| run: wget https://github.com/TileDB-Inc/TileDB/releases/download/2.17.1/tiledb-linux-x86_64-2.17.1-7c8c73b.tar.gz | |
| - name: Unpack release tarball | |
| run: | | |
| mkdir install-libtiledb | |
| tar xzf tiledb-linux-x86_64-*.tar.gz -C install-libtiledb | |
| ls -R install-libtiledb/ | |
| # - name: Configure TileDB | |
| # run: | | |
| # cmake -B build-libtiledb \ | |
| # -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| # -D TILEDB_VCPKG=OFF \ | |
| # -D TILEDB_WERROR=ON \ | |
| # -D TILEDB_SERIALIZATION=ON \ | |
| # -D CMAKE_BUILD_TYPE=Release | |
| # - name: Build TileDB | |
| # run: cmake --build build-libtiledb -j2 --config Release | |
| # - name: Install TileDB | |
| # run: cmake --build build-libtiledb --config Release --target install-tiledb | |
| - name: Upload libtiledb as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: | | |
| install-libtiledb/include/ | |
| install-libtiledb/lib*/ | |
| retention-days: 14 | |
| # - name: Test TileDB | |
| # run: cmake --build build-libtiledb --target check --config Release | |
| tiledb-py: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-Py | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-Py | |
| ref: ${{ github.event.inputs.TILEDB_PY_REF || 'dev' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade -r misc/requirements_ci.txt | |
| - name: Build TileDB-Py | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| python setup.py install --tiledb=$GITHUB_WORKSPACE/install-libtiledb/ | |
| # - name: Test TileDB-Py | |
| # run: | | |
| # python -c "import tiledb; print(tiledb.libtiledb.version())" | |
| # pytest -vv | |
| - name: Create a wheel | |
| run: python setup.py bdist_wheel --tiledb=$GITHUB_WORKSPACE/install-libtiledb/ | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| path: dist/tiledb-*-linux_x86_64.whl | |
| retention-days: 14 | |
| tiledb-r: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| _R_CHECK_FORCE_SUGGESTS_: "FALSE" | |
| steps: | |
| - name: Clone TileDB-R | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-R | |
| ref: ${{ github.event.inputs.TILEDB_R_REF || 'master' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Bootstrap | |
| run: ./.github/r-ci.sh bootstrap | |
| - name: Dependencies | |
| run: ./.github/r-ci.sh install_all | |
| - name: Build | |
| run: R CMD build --no-manual --no-build-vignettes . | |
| - name: Install | |
| run: | | |
| R CMD INSTALL --configure-args=--with-tiledb=$GITHUB_WORKSPACE/install-libtiledb --build tiledb_*.tar.gz | |
| - name: Upload R binary tarball as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledb-r-linux | |
| path: tiledb_*_R_x86_64-pc-linux-gnu.tar.gz | |
| retention-days: 14 | |
| - name: Test | |
| run: R -e 'tinytest::test_package("tiledb")' | |
| libtiledbvcf: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: main | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Configure libtiledbvcf | |
| run: | | |
| cmake -S libtiledbvcf -B build-libtiledbvcf \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledbvcf/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D TILEDB_WERROR=OFF | |
| - name: Build libtiledbvcf | |
| run: cmake --build build-libtiledbvcf -j2 --config Release | |
| - name: Install libtiledbvcf | |
| run: cmake --build build-libtiledbvcf --config Release --target install-libtiledbvcf | |
| - name: Upload libtiledbvcf as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: | | |
| install-libtiledbvcf/include/ | |
| install-libtiledbvcf/lib*/ | |
| retention-days: 14 | |
| - name: libtiledbvcf version | |
| run: ./install-libtiledbvcf/bin/tiledbvcf version | |
| - name: Confirm linking | |
| run: ldd install-libtiledbvcf/lib/libtiledbvcf.so | grep libtiledb.so | |
| - name: Install bcftools (for tests) | |
| run: sudo apt install bcftools | |
| - name: Test libtiledbvcf | |
| run: | | |
| make -j2 -C build-libtiledbvcf/libtiledbvcf tiledb_vcf_unit | |
| ./build-libtiledbvcf/libtiledbvcf/test/tiledb_vcf_unit | |
| # cli tests (require bcftools) | |
| # USAGE: run-cli-tests.sh <build-dir> <inputs-dir> | |
| libtiledbvcf/test/run-cli-tests.sh build-libtiledbvcf libtiledbvcf/test/inputs | |
| tiledbvcf-py: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbvcf, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbvcf/lib" | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: main | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbvcf | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: install-libtiledbvcf | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install \ | |
| dask \ | |
| distributed \ | |
| 'fsspec<2023.3.0' \ | |
| pyarrow \ | |
| pandas \ | |
| pybind11 \ | |
| pytest \ | |
| setuptools \ | |
| setuptools_scm \ | |
| setuptools_scm_git_archive \ | |
| wheel | |
| - name: Apply patch for -std=c++17 | |
| run: | | |
| wget https://raw.githubusercontent.com/TileDB-Inc/tiledb-vcf-feedstock/1d9e223924eb5b8dc9b00846411d0dbecf76cd8e/recipe/0001-Update-python-pybind11-to-use-c-17.patch | |
| git apply 0001-Update-python-pybind11-to-use-c-17.patch | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Build tiledbvcf-py | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| cd apis/python | |
| python setup.py install --libtiledbvcf=$GITHUB_WORKSPACE/install-libtiledbvcf/ | |
| - name: Install bcftools (for tests) | |
| run: sudo apt install bcftools | |
| - name: Test tiledbvcf-py | |
| run: | | |
| python -c "import tiledbvcf; print(tiledbvcf.version)" | |
| cd apis/python | |
| pytest | |
| - name: Create a wheel | |
| run: | | |
| cd apis/python | |
| python setup.py bdist_wheel --libtiledbvcf=$GITHUB_WORKSPACE/install-libtiledbvcf/ | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-tiledbvcf-py-linux | |
| path: dist/tiledbvcf-*-linux_x86_64.whl | |
| retention-days: 14 | |
| libtiledbsoma: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| env: | |
| TileDB_DIR: ${{ github.workspace }}/install-libtiledb | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: main | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Configure libtiledbsoma | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$GITHUB_WORKSPACE/install-libtiledbsoma/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| - name: Build libtiledbsoma | |
| run: cmake --build build-libtiledbsoma -j2 | |
| - name: Install libtiledbsoma | |
| run: cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| - name: Upload libtiledbsoma as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: | | |
| install-libtiledbsoma/include/ | |
| install-libtiledbsoma/lib*/ | |
| - name: Test libtiledbsoma | |
| run: | | |
| cmake --build build-libtiledbsoma/libtiledbsoma --target build_tests -j2 | |
| ctest --test-dir build-libtiledbsoma/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure | |
| tiledbsoma-py: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbsoma, tiledb-py] | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: main | |
| tiledbsoma-r: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb, libtiledbsoma, tiledb-r] | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: main | |
| tiledb-mariadb: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| steps: | |
| - name: Clone TileDB-MariaDB | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-MariaDB | |
| ref: master | |
| tiledb-go: | |
| runs-on: ubuntu-latest | |
| needs: [libtiledb] | |
| steps: | |
| - name: Clone TileDB-Go | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: TileDB-Inc/TileDB-Go | |
| ref: master | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Install dependencies | |
| run: go get -t . | |
| - name: Test TileDB-Go | |
| run: | | |
| export CPATH=$GITHUB_WORKSPACE/install-libtiledb/include | |
| export LIBRARY_PATH=$GITHUB_WORKSPACE/install-libtiledb/lib | |
| export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-libtiledb/lib | |
| go test -v ./... |