MLO-55 update to spark 4 and migrate to UV

.github/workflows/autodoc.yml

@@ -32,23 +32,20 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+            python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: |
-          poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: Publish Sphinx Pages
         run: |
-          cd docs && poetry run make html
+          cd docs && uv run make html
 
       - name: Setup Pages
         uses: actions/configure-pages@v5

.github/workflows/pull_rq.yml

@@ -16,21 +16,19 @@ jobs:
 
       - uses: actions/setup-java@v1
         with:
-          java-version: "8"
+          java-version: "21"
 
-      - uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+          python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: |
-          poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: ✅ Test with pytest
         run: |
-          poetry run pytest
+          uv run pytest

.github/workflows/release.yml

@@ -20,20 +20,19 @@ jobs:
       - name: Checkout Rialto
         uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+          python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: Build the wheel
-        run: poetry build
+        run: uv build
 
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -8,3 +8,4 @@ coverage.xml
 .ipynb_checkpoints/
 tmp
 dist
+.venv

.python-version

@@ -0,0 +1 @@
+3.12.3

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## 2.1.0 - 2025-10
+ ### General
+ - Updated python version to 3.12 and pyspark to 4.0
+ - Migrated from poetry to UV
+
 ## 2.0.11 - 2025-08-12
   ### Loader
   - optimized number of internal read operation when setting up PysparkFeatureLoader

docs/source/rialto.common.rst

@@ -4,6 +4,14 @@ rialto.common package
 Submodules
 ----------
 
+rialto.common.env\_yaml module
+----------------------------------
+
+.. automodule:: rialto.common.env_yaml
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 rialto.common.table\_reader module
 ----------------------------------
 

docs/source/rialto.jobs.rst

@@ -1,13 +1,49 @@
 rialto.jobs package
 ===================
 
-Subpackages
------------
+Submodules
+----------
 
-.. toctree::
-   :maxdepth: 4
+rialto.jobs.decorators module
+----------------------------------------
+
+.. automodule:: rialto.jobs.decorators
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.job\_base module
+---------------------------------------
+
+.. automodule:: rialto.jobs.job_base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.module\register module
+---------------------------------------
+
+.. automodule:: rialto.jobs.module_register
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.resolver module
+--------------------------------------
+
+.. automodule:: rialto.jobs.resolver
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.test\_utils module
+-----------------------------------------
+
+.. automodule:: rialto.jobs.test_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-   rialto.jobs.decorators
 
 Module contents
 ---------------

docs/source/rialto.loader.rst

@@ -12,13 +12,6 @@ rialto.loader.config\_loader module
    :undoc-members:
    :show-inheritance:
 
-rialto.loader.data\_loader module
----------------------------------
-
-.. automodule:: rialto.loader.data_loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
 
 rialto.loader.interfaces module
 -------------------------------

docs/source/rialto.runner.reporting.rst

@@ -0,0 +1,45 @@
+rialto.runner.reporting package
+=====================================
+
+Submodules
+----------
+
+rialto.runner.reporting.bookkeeper module
+-----------------------------------------
+
+.. automodule:: rialto.runner.reporting.bookkeeper
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.mailer module
+-------------------------------------
+
+.. automodule:: rialto.runner.reporting.mailer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.record module
+-------------------------------------
+
+.. automodule:: rialto.runner.reporting.record
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.tracker module
+--------------------------------------
+
+.. automodule:: rialto.runner.reporting.tracker
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: rialto.runner.reporting
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/rialto.runner.rst

@@ -1,6 +1,14 @@
 rialto.runner package
 =====================
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   rialto.runner.reporting
+
 Submodules
 ----------
 
@@ -12,6 +20,14 @@ rialto.runner.config\_loader module
    :undoc-members:
    :show-inheritance:
 
+rialto.runner.config\_overrides module
+--------------------------------------
+
+.. automodule:: rialto.runner.config_overrides
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 rialto.runner.date\_manager module
 ----------------------------------
 
@@ -36,18 +52,18 @@ rialto.runner.table module
    :undoc-members:
    :show-inheritance:
 
-rialto.runner.tracker module
-----------------------------
+rialto.runner.transformation module
+-----------------------------------
 
-.. automodule:: rialto.runner.tracker
+.. automodule:: rialto.runner.transformation
    :members:
    :undoc-members:
    :show-inheritance:
 
-rialto.runner.transformation module
+rialto.runner.utils module
 -----------------------------------
 
-.. automodule:: rialto.runner.transformation
+.. automodule:: rialto.runner.utils
    :members:
    :undoc-members:
    :show-inheritance: