Add support for marking and skipping slow tests, temporarily mark pydantic tests as slow

.github/workflows/unit-tests.yml

@@ -88,15 +88,17 @@ jobs:
       image: nemoci.azurecr.io/bionemo:${{ github.run_id }}
       options: --gpus all
       volumes:
-        - /home/azureuser/actions-runner-bionemo/cache:/root/.cache
+        - /home/azureuser/actions-runner-bionemo/cache:/github/home/.cache
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: ${{ github.run_id }}
 
       - name: Run tests
-        run: ./ci/scripts/run_pytest.sh --no-nbval
+        env:
+          BIONEMO_DATA_SOURCE: ngc
+        run: ./ci/scripts/run_pytest.sh --no-nbval --skip-slow
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
@@ -110,5 +112,14 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           working-directory: ${{ github.run_id }}
+
+  clean-up:
+    needs: run-tests
+    runs-on: self-hosted-nemo-gpus-1
+    if: ${{ always() }}
+    steps:
+      - name: clean up image
+        run: docker rmi nemoci.azurecr.io/bionemo:${{ github.run_id }}
+
 # TODO: exclude tests from base image; run tests from github workspace mounted in the image.
-# TODO: upload coverage report to codecov.io
+# TODO: figure out way of cleaning up working directory (requires sudo or for us to fix file ownership from release container)

ci/scripts/run_pytest.sh

@@ -27,6 +27,7 @@ Usage: $(basename "$0") [OPTIONS]
 Options:
     --skip-docs    Skip running tests in the docs directory
     --no-nbval     Skip jupyter notebook validation tests
+    --skip-slow    Skip tests marked as slow (@pytest.mark.slow)
 
 Note: Documentation tests (docs/) are only run when notebook validation
       is enabled (--no-nbval not set) and docs are not skipped
@@ -48,13 +49,15 @@ export BIONEMO_DATA_SOURCE PYTHONDONTWRITEBYTECODE PYTORCH_CUDA_ALLOC_CONF
 declare -a coverage_files
 SKIP_DOCS=false
 NO_NBVAL=false
+SKIP_SLOW=false
 error=false
 
 # Parse command line arguments
 while (( $# > 0 )); do
     case "$1" in
         --skip-docs) SKIP_DOCS=true ;;
         --no-nbval) NO_NBVAL=true ;;
+        --skip-slow) SKIP_SLOW=true ;;
         -h|--help) usage ;;
         *) echo "Unknown option: $1" >&2; usage 1 ;;
     esac
@@ -83,6 +86,7 @@ PYTEST_OPTIONS=(
     --cov-report=xml:coverage.xml
 )
 [[ "$NO_NBVAL" != true ]] && PYTEST_OPTIONS+=(--nbval-lax)
+[[ "$SKIP_SLOW" == true ]] && PYTEST_OPTIONS+=(-m "not slow")
 
 # Define test directories
 TEST_DIRS=(./sub-packages/bionemo-*/)

sub-packages/bionemo-esm2/tests/bionemo/esm2/scripts/test_pydantic_train.py

@@ -51,6 +51,8 @@ def dummy_parquet_train_val_inputs(tmp_path):
     return train_cluster_path, valid_cluster_path
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_pydantic_cli(dummy_protein_dataset, dummy_parquet_train_val_inputs, tmpdir):
     result_dir = tmpdir.mkdir("results")
     train_cluster_path, valid_cluster_path = dummy_parquet_train_val_inputs

sub-packages/bionemo-geneformer/tests/bionemo/geneformer/scripts/test_pydantic_train.py

@@ -42,6 +42,8 @@ def test_bionemo2_rootdir(data_path):
     assert data_path.is_dir(), f"Test data directory is supposed to be a directory.\n{data_error_str}"
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_cli_from_ckpt(tmpdir, data_path):
     # Same as test_pretrain, but includes a checkpoint to initialize from.
     result_dir = Path(tmpdir.mkdir("results"))
@@ -83,6 +85,8 @@ def test_pretrain_cli_from_ckpt(tmpdir, data_path):
     assert (result_dir / "test-experiment").exists(), "Could not find test experiment directory."
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_cli(tmpdir, data_path):
     """trains from scratch"""
     # data_path: Path = load("single_cell/testdata-20240506") / "cellxgene_2023-12-15_small" / "processed_data"
@@ -123,6 +127,8 @@ def test_pretrain_cli(tmpdir, data_path):
     assert (result_dir / "test-experiment").exists(), "Could not find test experiment directory."
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_finetune_cli(tmpdir, data_path):
     """Uses CLI to invoke the entrypoint"""
     # data_path: Path = load("single_cell/testdata-20240506") / "cellxgene_2023-12-15_small" / "processed_data"