Fixes to get a version that runs in production (#94)

Minor changes to get this running smoothly in the wild.

* Do not require clients to install the func-adl.ast package.
* Fix bug in how QMetaData updates items in its dict (overriden items were ignored!)
* Add a warning if there is a lambda that isn't scanned for type forwarding
* Fix bug where function names were being captured during lambda resolution
* Move to a single package rather than two (no more .ast). This is certianly a breaking change!

Author: gordonwatts

.github/workflows/ci.yml

@@ -5,31 +5,30 @@ on:
   pull_request:
   # Run daily at 0:01 UTC
   schedule:
-  - cron:  '1 0 * * *'
+    - cron: "1 0 * * *"
 
 jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.7
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        pip install --no-cache-dir -e .[complete]
-        pip list
-    - name: Lint with Flake8
-      run: |
-        flake8
-    - name: Build the pypi package
-      env:
-        func_adl_version: 1.0b10
-      run: |
-        python setup_func_adl.py sdist bdist_wheel
-        python setup_func_adl_ast.py sdist bdist_wheel
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install --no-cache-dir -e .[complete]
+          pip list
+      - name: Lint with Flake8
+        run: |
+          flake8
+      - name: Build the pypi package
+        env:
+          func_adl_version: 1.0b10
+        run: |
+          python setup.py sdist bdist_wheel
 
   test:
     runs-on: ${{ matrix.os }}
@@ -41,23 +40,23 @@ jobs:
         python-version: [3.7, 3.8, 3.9, "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        pip install --no-cache-dir -e .[complete]
-        pip list
-    - name: Test with pytest
-      run: |
-        python -m pytest -r sx
-    - name: Report coverage with Codecov
-      if: github.event_name == 'push' && matrix.python-version == 3.9 && matrix.os == 'ubuntu-latest'
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
-        flags: unittests
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install --no-cache-dir -e .[complete]
+          pip list
+      - name: Test with pytest
+        run: |
+          python -m pytest -r sx
+      - name: Report coverage with Codecov
+        if: github.event_name == 'push' && matrix.python-version == 3.9 && matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          flags: unittests

.github/workflows/pypi.yaml

@@ -9,34 +9,23 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.7
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-        # pip install --no-cache-dir -e .[complete]
-        # pip list
-    - name: Build the func_adl wheel
-      env:
-        func_adl_version: ${{ github.ref }}
-      run: |
-        python setup_func_adl.py sdist bdist_wheel
-    - name: Publish a func_adl to PyPI
-      uses: pypa/[email protected]
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password_func_adl }}
-    - name: Build the func_adl.ast wheel
-      env:
-        func_adl_version: ${{ github.ref }}
-      run: |
-        rm -rf dist/
-        python setup_func_adl_ast.py sdist bdist_wheel
-    - name: Publish a func_adl.ast to PyPI
-      uses: pypa/[email protected]
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password_func_adl_ast }}
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          # pip install --no-cache-dir -e .[complete]
+          # pip list
+      - name: Build the func_adl wheel
+        env:
+          func_adl_version: ${{ github.ref }}
+        run: |
+          python setup.py sdist bdist_wheel
+      - name: Publish a func_adl to PyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_password_func_adl }}

.vscode/settings.json

@@ -71,5 +71,6 @@
         "editor.codeActionsOnSave": {
             "source.organizeImports": true
         }
-    }
+    },
+    "python.formatting.provider": "black"
 }

func_adl/ast/__init__.py

@@ -1,6 +1,6 @@
-from .meta_data import extract_metadata  # NOQA
-from .call_stack import argument_stack, stack_frame  # NOQA
-from .func_adl_ast_utils import FuncADLNodeVisitor, function_call  # NOQA
 from .aggregate_shortcuts import aggregate_node_transformer  # NOQA
+from .call_stack import argument_stack, stack_frame  # NOQA
 from .func_adl_ast_utils import change_extension_functions_to_calls  # NOQA
+from .func_adl_ast_utils import FuncADLNodeVisitor, function_call  # NOQA
 from .function_simplifier import simplify_chained_calls  # NOQA
+from .meta_data import extract_metadata  # NOQA

func_adl/ast/meta_data.py

@@ -59,6 +59,7 @@ def extract_metadata(a: ast.AST) -> Tuple[ast.AST, List[Dict[str, str]]]:
 
 def remove_empty_metadata(a: ast.AST) -> ast.AST:
     """Returns a new ast with any empty `MetaData` clauses removed.
+    The old ast is not modified.
 
     Args:
         a (ast.AST): The AST of the query to clean up.
@@ -75,7 +76,7 @@ def visit_Call(self, node: ast.Call):
                 if len(n.args) == 2:
                     d = ast.literal_eval(n.args[1])
                     if isinstance(d, dict) and len(d) == 0:
-                        return self.visit(n.args[0])
+                        return n.args[0]
             return n
 
     return _cleaner().visit(a)
@@ -103,10 +104,14 @@ def found(self) -> Optional[Any]:
 
         def generic_visit(self, node: ast.AST):
             q_metadata = getattr(node, "_q_metadata", None)
+            found = False
             if q_metadata is not None:
                 if metadata_name in q_metadata:
+                    found = True
                     self._found = q_metadata[metadata_name]
-            return super().generic_visit(node)
+
+            if not found:
+                super().generic_visit(node)
 
     ds_f = _finder()
     ds_f.visit(q.query_ast)

func_adl/object_stream.py

@@ -199,16 +199,20 @@ def QMetaData(self, metadata: Dict[str, Any]) -> ObjectStream[T]:
         base_ast = self.query_ast
         for k, v in metadata.items():
             found_md = lookup_query_metadata(self, k)
+            add_md = False
             if found_md is None:
+                add_md = True
+            elif found_md != v:
+                logging.getLogger(__name__).info(
+                    f'Overwriting metadata "{k}" from its old value of "{found_md}" to "{v}"'
+                )
+                add_md = True
+            if add_md:
                 if first:
                     first = False
                     base_ast = self.MetaData({}).query_ast
                     base_ast._q_metadata = {}  # type: ignore
                 base_ast._q_metadata[k] = v  # type: ignore
-            elif found_md != v:
-                logging.getLogger(__name__).warning(
-                    f'Overwriting metadata "{k}" from its old value of "{found_md}" to "{v}"'
-                )
 
         return ObjectStream[T](base_ast, self.item_type)