chore: api changes for Datafusion 44 #7952
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: python_build | |
| on: | |
| merge_group: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| defaults: | |
| run: | |
| working-directory: ./python | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Check Python | |
| run: | | |
| uv pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions --system | |
| make check-python | |
| - name: Check Rust | |
| run: make check-rust | |
| test-minimal: | |
| name: Python Build (Python 3.9 PyArrow 16.0.0) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: 3.9 | |
| - name: Build and install deltalake | |
| run: | | |
| # Install minimum PyArrow version | |
| uv sync --extra devel --extra pandas | |
| uv pip install pyarrow==16.0.0 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| - name: Run tests | |
| run: make unit-test | |
| test: | |
| name: Python Build (Python 3.10 PyArrow latest) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Build and install deltalake | |
| run: make develop | |
| - name: Download Data Acceptance Tests (DAT) files | |
| run: make setup-dat | |
| - name: Run tests | |
| run: uv run pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules | |
| - name: Test without pandas | |
| run: | | |
| uv pip uninstall pandas | |
| uv run pytest -m "not pandas and not integration and not benchmark" | |
| uv pip install pandas | |
| test-pyspark: | |
| name: PySpark Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Build and install deltalake | |
| run: make develop-pyspark | |
| - name: Run tests | |
| run: make test-pyspark | |
| multi-python-running: | |
| name: Running with Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build and install deltalake | |
| run: | | |
| uv sync | |
| - name: Run deltalake | |
| run: | | |
| uv run python -c 'import deltalake' |