Upgrade datafusion #6591
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [master, nightly, develop] | |
| pull_request: | |
| branches: [develop] | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.11", "3.12"] | |
| defaults: | |
| run: | |
| shell: bash | |
| name: build - Python ${{ matrix.python-version }} (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BUILD_MODE: debug | |
| RUST_BACKTRACE: 1 | |
| services: | |
| redis: | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: pass | |
| POSTGRES_DB: nautilus | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: Free disk space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| android: false | |
| dotnet: false | |
| haskell: false | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Install runner dependencies | |
| run: sudo apt-get install -y curl clang git libssl-dev make pkg-config | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust toolchain | |
| run: | | |
| rustup toolchain add --profile minimal stable --component clippy,rustfmt | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get Python version | |
| run: | | |
| version=$(bash scripts/python-version.sh) | |
| echo "PYTHON_VERSION=$version" >> $GITHUB_ENV | |
| - name: Get Poetry version from poetry-version | |
| run: | | |
| version=$(cat poetry-version) | |
| echo "POETRY_VERSION=$version" >> $GITHUB_ENV | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| - name: Install build dependencies | |
| run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec | |
| # ta-lib Python install currently broken | |
| # https://github.com/TA-Lib/ta-lib-python/issues/655 | |
| # - name: Install TA-Lib (Linux) | |
| # run: | | |
| # make install-talib | |
| # poetry run pip install setuptools numpy==1.26.4 ta-lib | |
| - name: Cached pre-commit | |
| id: cached-pre-commit | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Cached cargo | |
| id: cached-cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Set poetry cache-dir | |
| run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV | |
| - name: Cached poetry | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.POETRY_CACHE_DIR }} | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-poetry-${{ hashFiles('**/poetry.lock') }} | |
| - name: Cached test data | |
| id: cached-testdata-large | |
| uses: actions/cache@v4 | |
| with: | |
| path: tests/test_data/large | |
| key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }} | |
| restore-keys: ${{ runner.os }}-large-files- | |
| - name: Run pre-commit | |
| run: | | |
| # pre-commit run --hook-stage manual gitlint-ci | |
| pre-commit run --all-files | |
| - name: Install Nautilus CLI and run init postgres | |
| run: | | |
| make install-cli | |
| nautilus database init --schema ${{ github.workspace }}/schema | |
| env: | |
| POSTGRES_HOST: localhost | |
| POSTGRES_PORT: 5432 | |
| POSTGRES_USERNAME: postgres | |
| POSTGRES_PASSWORD: pass | |
| POSTGRES_DATABASE: nautilus | |
| - name: Run nautilus_core cargo tests (Linux) | |
| run: | | |
| if ! command -v cargo-nextest &> /dev/null | |
| then | |
| echo "cargo-nextest not found, installing..." | |
| cargo install cargo-nextest | |
| else | |
| echo "cargo-nextest is already installed" | |
| fi | |
| make cargo-test | |
| - name: Run tests (Linux) | |
| run: | | |
| make pytest | |
| make test-examples | |
| # Run codspeed for latest python-version only | |
| - name: Run benchmarks (Linux) | |
| if: ${{ matrix.python-version == '3.12' }} | |
| uses: CodSpeedHQ/action@v3 | |
| with: | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| run: make test-performance | |
| build-windows: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ["3.11", "3.12"] | |
| defaults: | |
| run: | |
| shell: bash | |
| name: build - Python ${{ matrix.python-version }} (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BUILD_MODE: debug | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Free disk space (Windows) | |
| run: | | |
| rm -rf "/c/Program Files/dotnet" | |
| rm -rf "/c/Program Files (x86)/Microsoft Visual Studio/2019" | |
| - name: Set up Rust toolchain | |
| run: | | |
| rustup toolchain add --profile minimal stable --component clippy,rustfmt | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get Python version | |
| run: | | |
| version=$(bash scripts/python-version.sh) | |
| echo "PYTHON_VERSION=$version" >> $GITHUB_ENV | |
| - name: Get Poetry version from poetry-version | |
| run: | | |
| version=$(cat poetry-version) | |
| echo "POETRY_VERSION=$version" >> $GITHUB_ENV | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| - name: Install build dependencies | |
| run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec | |
| - name: Cached pre-commit | |
| id: cached-pre-commit | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Cached cargo | |
| id: cached-cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Set poetry cache-dir | |
| run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV | |
| - name: Cached poetry | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.POETRY_CACHE_DIR }} | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-poetry-${{ hashFiles('**/poetry.lock') }} | |
| - name: Cached test data | |
| id: cached-testdata-large | |
| uses: actions/cache@v4 | |
| with: | |
| path: tests/test_data/large | |
| key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }} | |
| restore-keys: ${{ runner.os }}-large-files- | |
| - name: Run pre-commit | |
| run: | | |
| # pre-commit run --hook-stage manual gitlint-ci | |
| pre-commit run --all-files | |
| # Run tests without parallel build (avoids linker errors) | |
| - name: Run tests (Windows) | |
| run: | | |
| poetry install --with test --all-extras | |
| poetry run pytest --ignore=tests/performance_tests --new-first --failed-first | |
| env: | |
| PARALLEL_BUILD: false | |
| build-macos: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest] | |
| python-version: ["3.11", "3.12"] | |
| defaults: | |
| run: | |
| shell: bash | |
| name: build - Python ${{ matrix.python-version }} (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BUILD_MODE: debug | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Free disk space (macOS) | |
| run: | | |
| sudo rm -rf ~/Library/Caches/* | |
| sudo rm -rf ~/Library/Developer/Xcode/DerivedData/* | |
| sudo rm -rf /Library/Developer/CommandLineTools | |
| - name: Set up Rust toolchain | |
| run: | | |
| rustup toolchain add --profile minimal stable --component clippy,rustfmt | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get Python version | |
| run: | | |
| version=$(bash scripts/python-version.sh) | |
| echo "PYTHON_VERSION=$version" >> $GITHUB_ENV | |
| - name: Get Poetry version from poetry-version | |
| run: | | |
| version=$(cat poetry-version) | |
| echo "POETRY_VERSION=$version" >> $GITHUB_ENV | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: ${{ env.POETRY_VERSION }} | |
| - name: Install build dependencies | |
| run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec | |
| - name: Cached pre-commit | |
| id: cached-pre-commit | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Cached cargo | |
| id: cached-cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Set poetry cache-dir | |
| run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV | |
| - name: Cached poetry | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.POETRY_CACHE_DIR }} | |
| key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-poetry-${{ hashFiles('**/poetry.lock') }} | |
| - name: Cached test data | |
| id: cached-testdata-large | |
| uses: actions/cache@v4 | |
| with: | |
| path: tests/test_data/large | |
| key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }} | |
| restore-keys: ${{ runner.os }}-large-files- | |
| - name: Run pre-commit | |
| run: | | |
| # pre-commit run --hook-stage manual gitlint-ci | |
| pre-commit run --all-files | |
| - name: Run nautilus_core cargo tests (macOS) | |
| run: | | |
| if ! command -v cargo-nextest &> /dev/null | |
| then | |
| echo "cargo-nextest not found, installing..." | |
| cargo install cargo-nextest | |
| else | |
| echo "cargo-nextest is already installed" | |
| fi | |
| make cargo-test | |
| - name: Run tests (macOS) | |
| run: | | |
| make pytest | |
| make test-examples |