Merge branch 'main' into spark-integration-test

.github/CODEOWNERS

@@ -1,4 +1,4 @@
-crates/ @wjones127 @roeap @rtyler
+crates/ @wjones127 @roeap @rtyler @hntd187 @ion-elgreco
 delta-inspect/ @wjones127 @rtyler
 proofs/ @houqp
 python/ @wjones127 @fvaleye @roeap @ion-elgreco

.github/actions/setup-env/action.yml

@@ -0,0 +1,34 @@
+name: "Setup Python and Rust Environment"
+description: "Set up Python, virtual environment, and Rust toolchain"
+
+inputs:
+
+  python-version:
+    description: "The Python version to set up"
+    required: true
+    default: "3.10"
+
+  rust-toolchain:
+    description: "The Rust toolchain to set up"
+    required: true
+    default: "stable"
+
+runs:
+  using: "composite"
+
+  steps:
+
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Rust toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: default
+        toolchain: ${{ inputs.rust-toolchain }}
+        override: true
+        components: rustfmt, clippy
+
+    - uses: Swatinem/rust-cache@v2

.github/workflows/build.yml

@@ -5,6 +5,7 @@ on:
     branches: [main, "rust-v*"]
   pull_request:
     branches: [main, "rust-v*"]
+  merge_group:
 
 jobs:
   format:
@@ -28,7 +29,6 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-          - macos-11
           - windows-latest
     runs-on: ${{ matrix.os }}
 
@@ -42,16 +42,14 @@ jobs:
           toolchain: stable
           override: true
 
-      - uses: Swatinem/rust-cache@v2
-
       - name: build and lint with clippy
-        run: cargo clippy --features azure,datafusion,s3,gcs,glue --tests
+        run: cargo clippy --features azure,datafusion,s3,gcs,glue,hdfs --tests
 
       - name: Spot-check build for native-tls features
         run: cargo clippy --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue --tests
 
       - name: Check docs
-        run: cargo doc --features azure,datafusion,s3,gcs,glue
+        run: cargo doc --features azure,datafusion,s3,gcs,glue,hdfs
 
       - name: Check no default features (except rustls)
         run: cargo check --no-default-features --features rustls
@@ -62,7 +60,6 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-          - macos-11
           - windows-latest
     runs-on: ${{ matrix.os }}
     env:
@@ -82,8 +79,6 @@ jobs:
           toolchain: "stable"
           override: true
 
-      - uses: Swatinem/rust-cache@v2
-
       - name: Run tests
         run: cargo test --verbose --features datafusion,azure
 
@@ -118,28 +113,24 @@ jobs:
           toolchain: stable
           override: true
 
-      # - uses: actions/setup-java@v3
-      #   with:
-      #     distribution: "zulu"
-      #     java-version: "17"
-
-      # - uses: beyondstorage/setup-hdfs@master
-      #   with:
-      #     hdfs-version: "3.3.2"
-
-      # - name: Set Hadoop env
-      #   run: |
-      #     echo "CLASSPATH=$CLASSPATH:`hadoop classpath --glob`" >> $GITHUB_ENV
-      #     echo "LD_LIBRARY_PATH=$JAVA_HOME/lib/server" >> $GITHUB_ENV
+      # Install Java and Hadoop for HDFS integration tests
+      - uses: actions/setup-java@v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
 
-      - uses: Swatinem/rust-cache@v2
+      - name: Download Hadoop
+        run: |
+          wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz
+          tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE
+          echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH
 
       - name: Start emulated services
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Run tests with rustls (default)
         run: |
-          cargo test --features integration_test,azure,s3,gcs,datafusion
+          cargo test --features integration_test,azure,s3,gcs,datafusion,hdfs
 
       - name: Run tests with native-tls
         run: |

.github/workflows/dev_pr.yml

@@ -2,6 +2,7 @@ name: dev_pr
 
 # Trigger whenever a PR is changed (title as well as new / changed commits)
 on:
+  merge_group:
   pull_request_target:
     types:
       - opened

.github/workflows/docs.yml

@@ -1,6 +1,7 @@
 name: Build (and maybe release) the documentation
 
 on:
+  merge_group:
   pull_request:
     paths:
       - python/**
@@ -31,9 +32,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: psf/black@stable
-        with:
-          src: docs/src/python
+      - run: |
+          cd docs
+          make check
 
   build-deploy:
     needs:
@@ -47,25 +48,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.10'
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Build and install deltalake
         run: |
           cd python
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make ${{ env.BUILD_ARGS }}
 

.github/workflows/python_build.yml

@@ -1,6 +1,7 @@
 name: python_build
 
 on:
+  merge_group:
   push:
     branches: [main]
   pull_request:
@@ -15,23 +16,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Check Python
         run: |
-          pip install ruff black mypy types-dataclasses typing-extensions
+          python -m venv venv
+          source venv/bin/activate
+          pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions
           make check-python
 
-      - name: Install minimal stable with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: stable
-          override: true
-
       - name: Check Rust
         run: make check-rust
 
@@ -45,24 +40,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v2
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
         with:
           python-version: 3.8
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make setup
           # Install minimum PyArrow version
@@ -89,26 +74,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v3
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Start emulated services
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make develop
 
@@ -137,23 +111,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Build deltalake in release mode
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           MATURIN_EXTRA_ARGS=--release make develop
 
@@ -187,18 +150,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v3
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - uses: actions/setup-java@v2
         with:
@@ -207,8 +160,7 @@ jobs:
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make develop-pyspark
 
@@ -231,15 +183,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make setup
           maturin develop

.github/workflows/python_release.yml

@@ -35,7 +35,7 @@ jobs:
       fail-fast: false
       matrix:
         target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    runs-on: macos-12
+    runs-on: macos-14
     steps:
       - uses: actions/checkout@v3
 

.gitignore

@@ -11,6 +11,8 @@ tlaplus/*.toolbox/*/[0-9]*-[0-9]*-[0-9]*-[0-9]*-[0-9]*-[0-9]*/
 /.idea
 .vscode
 .env
+.venv
+venv
 **/.DS_Store
 **/.python-version
 .coverage
@@ -20,6 +22,7 @@ __blobstorage__
 .githubchangeloggenerator.cache.log
 .githubchangeloggenerator.cache/
 .githubchangeloggenerator*
+data
 
 # Add all Cargo.lock files except for those in binary crates
 Cargo.lock