chore: refactor makefile and docs appendix

.github/workflows/build.yaml

@@ -21,7 +21,7 @@ jobs:
         pipx install poetry
     - name: Install dependencies
       run: |
-        poetry install
+        make install
     - name: Run tests
       run: |
-        poetry run pytest -s -x
+        make test

.github/workflows/docs.yaml

@@ -25,4 +25,5 @@ jobs:
           restore-keys: |
             mkdocs-material-
       - run: pip install mkdocs-material 
+      - run: make gen-docs 
       - run: mkdocs gh-deploy --force

.github/workflows/e2e.yaml

@@ -21,17 +21,17 @@ jobs:
         pipx install poetry
     - name: Install dependencies
       run: |
-        poetry install
+        make install
     - name: Start Kind Cluster
       uses: helm/kind-action@v1
       with:
         cluster_name: kind
     - name: Start distribution-registry
       run: |
         ./e2e/deploy_distribution_registry.sh
-    - name: Run tests
+    - name: Run E2E tests
       run: |
-        poetry run pytest --e2e -s -x
+        make test-e2e
   e2e-zot:
     runs-on: ubuntu-latest
     steps:
@@ -46,14 +46,14 @@ jobs:
         pipx install poetry
     - name: Install dependencies
       run: |
-        poetry install
+        make install
     - name: Start Kind Cluster
       uses: helm/kind-action@v1
       with:
         cluster_name: kind
     - name: Start Zot
       run: |
         ./e2e/deploy_zot.sh
-    - name: Run tests
+    - name: Run E2E tests
       run: |
-        poetry run pytest --e2e -s -x
+        make test-e2e

.gitignore

@@ -7,4 +7,9 @@ tmp
 efi-variable-store
 *.onnx
 mr-integration/config/ml-metadata/metadata.sqlite.db
+
+# Generated python library wheels
 dist
+
+# Mkdocs
+site

Makefile

@@ -1,5 +1,28 @@
 
+.PHONY: install
+install:
+	poetry install
+
+.PHONY: gen-docs
+gen-docs: install
+	poetry run python docs/appendixes/gen-appendix-links.py docs/
+
 .PHONY: docs-live
-docs-live:
+docs-live: gen-docs
 	docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material
 
+.PHONY: docs-build
+docs-build: gen-docs
+	docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material build
+
+.PHONY: build
+build: install
+	poetry build
+
+.PHONY: test
+test:
+	poetry run pytest -s -x -rA
+
+.PHONY: test-e2e
+test-e2e:
+	poetry run pytest --e2e -s -x -rA

docs/appendixes/appendix-links.md

@@ -0,0 +1,14 @@
+<!-- this file is generated by toolings -->
+# Appendix "All links"
+
+This is a collection of all the links from this website:
+
+- https://docs.sigstore.dev/
+- https://github.com/kubeflow/community/pull/682#issuecomment-2130072374
+- https://github.com/kubernetes/enhancements/pull/4639
+- https://github.com/lampajr/oci-storage-initializer/blob/main/GET_STARTED.md
+- https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
+- https://www.cncf.io/projects/confidential-containers
+- https://www.kubeflow.org/docs/components/model-registry
+- https://www.openpolicyagent.org/
+

docs/appendixes/gen-appendix-links.py

@@ -0,0 +1,42 @@
+from markdown_it.tree import SyntaxTreeNode
+from markdown_it import MarkdownIt
+
+import os
+import fnmatch
+import click
+
+@click.command()
+@click.argument('dir', required=True)
+def parse_markdown_files(dir):
+    all_href = get_all_hrefs(dir)
+    with open(f"{dir}/appendixes/appendix-links.md", "w") as appendix:
+        appendix.write("""<!-- this file is generated by toolings -->
+# Appendix "All links"
+
+This is a collection of all the links from this website:
+
+"""
+        )
+        for e in all_href:
+            appendix.write(f"- {e}\n")
+        appendix.write("\n")
+
+def get_all_hrefs(dir):
+    md = MarkdownIt("commonmark")
+    all_href = []
+    for root, _, files in os.walk(dir):
+        for filename in fnmatch.filter(files, '*.md'):
+            if filename == "appendix-links.md":
+                continue # skip appendix-links file itself
+            with open(f"{root}/{filename}", "r") as f:
+                tokens = md.parse(f.read())
+                node = SyntaxTreeNode(tokens)
+                for n in node.walk():
+                    if n.type == "link":
+                        all_href.append(n.attrs["href"])
+    all_href.sort()
+    print(all_href)
+    return all_href
+
+if __name__ == '__main__':
+    parse_markdown_files()

mkdocs.yml

@@ -19,6 +19,8 @@ nav:
     - 'Demo 3: Signature and Attestation': 'demos/demo3.md'
   - Use Cases: 'use-cases.md'
   - Glossary: 'glossary.md'
+  - Appendixes:
+    - 'All links': 'appendixes/appendix-links.md'
 extra:
   social:
     - icon: fontawesome/brands/github 

pyproject.toml

@@ -18,6 +18,7 @@ pytest-mock = "^3.14.0"
 jq = "^1.7.0"
 scikit-learn = "^1.5.0"
 ipykernel = "^6.29.4"
+markdown-it-py = "^3.0.0"
 
 [tool.poetry.scripts]
 omlmd = "omlmd.cli:cli"

tests/conftest.py

@@ -3,12 +3,12 @@
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--e2e"):
-        skip_not_e2e = pytest.mark.skip(reason="skipping non-e2e tests")
+        skip_not_e2e = pytest.mark.skip(reason="skipping non-e2e tests; opt-out of --e2e option to run.")
         for item in items:
             if "e2e" not in item.keywords:
                 item.add_marker(skip_not_e2e)
         return
-    skip_e2e = pytest.mark.skip(reason="test requires --e2e option to run")
+    skip_e2e = pytest.mark.skip(reason="this is an end-to-end test, requires explicit opt-in --e2e option to run.")
     for item in items:
         if "e2e" in item.keywords:
             item.add_marker(skip_e2e)