Release/2.1.0

.github/workflows/autodoc.yml

@@ -32,23 +32,20 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+            python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: |
-          poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: Publish Sphinx Pages
         run: |
-          cd docs && poetry run make html
+          cd docs && uv run make html
 
       - name: Setup Pages
         uses: actions/configure-pages@v5

.github/workflows/pull_rq.yml

@@ -16,21 +16,19 @@ jobs:
 
       - uses: actions/setup-java@v1
         with:
-          java-version: "8"
+          java-version: "21"
 
-      - uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+          python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: |
-          poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: ✅ Test with pytest
         run: |
-          poetry run pytest
+          uv run pytest

.github/workflows/release.yml

@@ -20,20 +20,19 @@ jobs:
       - name: Checkout Rialto
         uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
+      - name: "Set up Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10.6"
+          python-version-file: ".python-version"
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.5.1
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
 
-      - name: Install dependencies
-        run: poetry install
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
 
       - name: Build the wheel
-        run: poetry build
+        run: uv build
 
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -8,3 +8,4 @@ coverage.xml
 .ipynb_checkpoints/
 tmp
 dist
+.venv

.python-version

@@ -0,0 +1 @@
+3.12.3

CHANGELOG.md

@@ -1,6 +1,15 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## 2.1.0 - 2025-10
+ ### General
+ - Updated python version to 3.12 and pyspark to 4.0
+ - Migrated from poetry to UV
+ ### Runner
+ - Added merge_schema manual override option
+ ### Maker
+ - Added another feature decorator _@template_ to support feature to text conversion
+
 ## 2.0.11 - 2025-08-12
   ### Loader
   - optimized number of internal read operation when setting up PysparkFeatureLoader

README.md

@@ -57,6 +57,7 @@ This behavior can be modified by various parameters and switches available.
 * **op** - run only selected operation / pipeline
 * **skip_dependencies** - ignore dependency checks and run all jobs
 * **overrides** - dictionary of overrides for the configuration
+* **merge_schema** - write output dataframe with mergeSchema option enabled
 
 
 Transformations are not included in the runner itself, it imports them dynamically according to the configuration, therefore it's necessary to have them locally installed.
@@ -282,6 +283,22 @@ def RECENCY() -> Column:
     return F.months_between(F.lit(rfm.FeatureMaker.make_date), F.col("DATE"))
 ```
 
+#### @template
+Provides an option to pass a template string that's to be used to create a text feature, this template is saved in [metadata](#metadata).
+
+```python
+import pyspark.sql.functions as F
+import rialto.maker as rfm
+from pyspark.sql import Column
+from rialto.metadata import ValueType as VT
+
+@rfm.feature(VT.numerical)
+@rfm.desc("Age of customer")
+@rfm.template("Customer is $X years old")
+def AGE() -> Column:
+    return F.col("AGE")
+```
+
 #### @param
 Inspired by @pytest.mark.parametrize, it has similar interface and fulfills the same role. It allows you to invoke the feature function multiple times with different values of the parameter.
 If multiple @params are used, the number of final features will be a product of all parameters. The feature function has to expect a parameter with the same name as the @params name.

docs/source/conf.py

@@ -28,7 +28,7 @@
 project = "rialto"
 copyright = "2022, Marek Dobransky"
 author = "Marek Dobransky"
-release = "2.0.11"
+release = "2.1.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

docs/source/rialto.common.rst

@@ -4,6 +4,14 @@ rialto.common package
 Submodules
 ----------
 
+rialto.common.env\_yaml module
+----------------------------------
+
+.. automodule:: rialto.common.env_yaml
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 rialto.common.table\_reader module
 ----------------------------------
 

docs/source/rialto.jobs.rst

@@ -1,13 +1,49 @@
 rialto.jobs package
 ===================
 
-Subpackages
------------
+Submodules
+----------
 
-.. toctree::
-   :maxdepth: 4
+rialto.jobs.decorators module
+----------------------------------------
+
+.. automodule:: rialto.jobs.decorators
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.job\_base module
+---------------------------------------
+
+.. automodule:: rialto.jobs.job_base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.module\register module
+---------------------------------------
+
+.. automodule:: rialto.jobs.module_register
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.resolver module
+--------------------------------------
+
+.. automodule:: rialto.jobs.resolver
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.jobs.test\_utils module
+-----------------------------------------
+
+.. automodule:: rialto.jobs.test_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-   rialto.jobs.decorators
 
 Module contents
 ---------------

docs/source/rialto.loader.rst

@@ -12,13 +12,6 @@ rialto.loader.config\_loader module
    :undoc-members:
    :show-inheritance:
 
-rialto.loader.data\_loader module
----------------------------------
-
-.. automodule:: rialto.loader.data_loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
 
 rialto.loader.interfaces module
 -------------------------------

docs/source/rialto.runner.reporting.rst

@@ -0,0 +1,45 @@
+rialto.runner.reporting package
+=====================================
+
+Submodules
+----------
+
+rialto.runner.reporting.bookkeeper module
+-----------------------------------------
+
+.. automodule:: rialto.runner.reporting.bookkeeper
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.mailer module
+-------------------------------------
+
+.. automodule:: rialto.runner.reporting.mailer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.record module
+-------------------------------------
+
+.. automodule:: rialto.runner.reporting.record
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rialto.runner.reporting.tracker module
+--------------------------------------
+
+.. automodule:: rialto.runner.reporting.tracker
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: rialto.runner.reporting
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/rialto.runner.rst

@@ -1,6 +1,14 @@
 rialto.runner package
 =====================
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   rialto.runner.reporting
+
 Submodules
 ----------
 
@@ -12,6 +20,14 @@ rialto.runner.config\_loader module
    :undoc-members:
    :show-inheritance:
 
+rialto.runner.config\_overrides module
+--------------------------------------
+
+.. automodule:: rialto.runner.config_overrides
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 rialto.runner.date\_manager module
 ----------------------------------
 
@@ -36,18 +52,18 @@ rialto.runner.table module
    :undoc-members:
    :show-inheritance:
 
-rialto.runner.tracker module
-----------------------------
+rialto.runner.transformation module
+-----------------------------------
 
-.. automodule:: rialto.runner.tracker
+.. automodule:: rialto.runner.transformation
    :members:
    :undoc-members:
    :show-inheritance:
 
-rialto.runner.transformation module
+rialto.runner.utils module
 -----------------------------------
 
-.. automodule:: rialto.runner.transformation
+.. automodule:: rialto.runner.utils
    :members:
    :undoc-members:
    :show-inheritance: