Added tests (#43)

* Added tests * Typo * Add sh package * Add bash and conda-lock * Fix tests * Move test flag up * More fixes to local datastore use * Reduce number of tests to try to limit disk usage * More attempts to fix out of space issue * Fix micromamba tests on linux; fail fast to understand failures on mamba/conda * Remove installed environments and packages to try to keep saving more space * Typo * Long standing typo in config file * Kill me now * Only look in created envs; reduce parallelism to reduce disk size used * Run black with new version * More reduction in parallelism * Move chdir out * Break up large test case to save on disk space * Make test smaller
Netflix · Aug 14, 2024 · 3d51d04 · 3d51d04
1 parent 9e02bbd
commit 3d51d04
Show file tree

Hide file tree

Showing 42 changed files with 729 additions and 95 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,56 @@
+name: test
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+    - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+        python-version: [ "3.8", "3.10", "3.12" ]
+        resolver: [ mamba, conda, micromamba ]
+    env:
+      METAFLOW_CONDA_DEPENDENCY_RESOLVER: ${{ matrix.resolver }}
+      METAFLOW_CONDA_TEST: 1
+      METAFLOW_DATASTORE_SYSROOT_LOCAL: .metaflow
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1.9.0
+        with:
+          micromamba-version: latest
+          environment-file: dev-env.yml
+          init-shell: bash
+          create-args: >-
+            python=${{ matrix.python-version }}
+
+      - name: install nflx-extension
+        shell: bash -eo pipefail -l {0}
+        run: |
+          which pip
+          pip install -e . --force-reinstall -U
+      
+      - name: install bash
+        if: runner.os == 'macOS'
+        run: brew install bash
+
+      - name: run test
+        shell: bash -eo pipefail -l {0}
+        run: |
+          set -x
+          which pytest
+          mkdir .metaflow
+          pytest -n 4 tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: check-yaml
+      - id: check-json
+  - repo: https://github.com/ambv/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+        language_version: python3
+        exclude: "^/metaflow_extensions/netflix_ext/vendor/"
+        args: [-t, py34, -t, py35, -t, py36, -t, py37, -t, py38, -t, py39, -t, py310, -t, py311, -t, py312]
+
+
diff --git a/dev-env.yml b/dev-env.yml
@@ -0,0 +1,16 @@
+name: nflxext-dev
+category: dev
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - pytest
+  - pytest-xdist
+  - pytest-timeout
+  - wheel
+  - mamba
+  - conda
+  - conda-lock
+  - conda-package-handling
+  - pip:
+    - sh
diff --git a/metaflow_extensions/netflix_ext/cmd/debug/debug_script_generator.py b/metaflow_extensions/netflix_ext/cmd/debug/debug_script_generator.py
@@ -364,15 +364,17 @@ def generate_notebook_json(
         # Create an empty notebook
         nb = {
             "cells": [],
-            "metadata": {}
-            if kernel_def is None
-            else {
-                "kernelspec": {
-                    "name": kernel_def[0],
-                    "display_name": kernel_def[1],
-                    "language": "python",
+            "metadata": (
+                {}
+                if kernel_def is None
+                else {
+                    "kernelspec": {
+                        "name": kernel_def[0],
+                        "display_name": kernel_def[1],
+                        "language": "python",
+                    }
                 }
-            },
+            ),
             "nbformat": 4,
             "nbformat_minor": 4,
         }
@@ -458,9 +460,11 @@ def generate_notebook_cells(
                 "format": "markdown",
                 "content": Constants.JUPYTER_TITLE_MARKDOWN.format(
                     TaskPathSpec=self.task_pathspec,
-                    DebugType="Post-Execution State"
-                    if self.inspect
-                    else "Pre-Execution State",
+                    DebugType=(
+                        "Post-Execution State"
+                        if self.inspect
+                        else "Pre-Execution State"
+                    ),
                 ),
             }
         ]

diff --git a/metaflow_extensions/netflix_ext/cmd/environment/environment_cmd.py b/metaflow_extensions/netflix_ext/cmd/environment/environment_cmd.py
@@ -21,6 +21,7 @@
 from metaflow.plugins import DATASTORES
 from metaflow.metaflow_config import (
     CONDA_ALL_ARCHS,
+    CONDA_TEST,
     CONDA_SYS_DEPENDENCIES,
     DEFAULT_DATASTORE,
     DEFAULT_METADATA,
@@ -359,7 +360,7 @@ def create(
         )
         resolver.resolve_environments(obj.echo)
         update_envs = []  # type: List[ResolvedEnvironment]
-        if obj.datastore_type != "local":
+        if obj.datastore_type != "local" or CONDA_TEST:
             # We may need to update caches
             # Note that it is possible that something we needed to resolve, we don't need
             # to cache (if we resolved to something already cached).
@@ -823,9 +824,9 @@ def resolve(
             sources,
             new_extras,
             base_env=base_env if using_str else None,
-            base_from_full_id=base_env.is_info_accurate
-            if base_env and using_str
-            else False,
+            base_from_full_id=(
+                base_env.is_info_accurate if base_env and using_str else False
+            ),
             local_only=local_only,
             force=force,
             force_co_resolve=len(archs) > 1,
@@ -881,7 +882,7 @@ def resolve(
     # If this is not a dry-run, we cache the environments and write out the resolved
     # information
     update_envs = []  # type: List[ResolvedEnvironment]
-    if obj.datastore_type != "local":
+    if obj.datastore_type != "local" or CONDA_TEST:
         # We may need to update caches
         # Note that it is possible that something we needed to resolve, we don't need
         # to cache (if we resolved to something already cached).
@@ -1215,9 +1216,7 @@ def _parse_yml_file(
                     to_update = (
                         conda_deps
                         if mode == "deps"
-                        else pypi_deps
-                        if mode == "pypi_deps"
-                        else sys_deps
+                        else pypi_deps if mode == "pypi_deps" else sys_deps
                     )
                     splits = YML_SPLIT_LINE.split(line.replace(" ", ""), maxsplit=1)
                     if len(splits) == 1:

diff --git a/metaflow_extensions/netflix_ext/cmd/mfextinit_netflixext.py b/metaflow_extensions/netflix_ext/cmd/mfextinit_netflixext.py
@@ -1,4 +1,4 @@
 CMDS_DESC = [
     ("debug", ".debug.debug_cmd.cli"),
-    ("environment", ".environment.environment_cmd.cli")
+    ("environment", ".environment.environment_cmd.cli"),
 ]
diff --git a/metaflow_extensions/netflix_ext/config/mfextinit_netflixext.py b/metaflow_extensions/netflix_ext/config/mfextinit_netflixext.py
@@ -8,6 +8,9 @@
 from metaflow.metaflow_config_funcs import from_conf, get_validate_choice_fn
 
 
+# Set to true if running the tests with a local datastore
+CONDA_TEST = from_conf("CONDA_TEST", False)
+
 CONDA_S3ROOT = from_conf(
     "CONDA_S3ROOT",
     os.path.join(DATASTORE_SYSROOT_S3, "conda_env") if DATASTORE_SYSROOT_S3 else None,
@@ -36,7 +39,7 @@
     ),
 )
 
-CONDA_MAGIC_FILE_V2 = "conda_v2.cnd"
+CONDA_MAGIC_FILE_V2 = from_conf("CONDA_MAGIC_FILE_V2", "conda_v2.cnd")
 
 # Use an alternate dependency resolver for conda packages instead of conda
 # Mamba promises faster package dependency resolution times, which
@@ -65,7 +68,7 @@
 CONDA_LOCK_TIMEOUT = from_conf("CONDA_LOCK_TIMEOUT", 300)
 
 # Location within CONDA_<DS>ROOT of the packages directory
-CONDA_PACKAGES_DIRNAME = from_conf("ENV_PACKAGES_DIRNAME", "packages")
+CONDA_PACKAGES_DIRNAME = from_conf("CONDA_PACKAGES_DIRNAME", "packages")
 # Ditto for the envs directory
 CONDA_ENVS_DIRNAME = from_conf("CONDA_ENVS_DIRNAME", "envs")