CI for v3.0.1

.github/workflows/publish_v3.yml

@@ -10,67 +10,134 @@ jobs:
   build:
     runs-on: ubuntu-latest
     container: python:3
+    env:
+      PARSER_OUTPUT_DIR: "__parser_output"  # temp dir for output from spec-parser
+      RDF_BASE_DIR: ""                      # change to "rdf" in new structure
+      MKDOCS_BASE_DIR: ""                   # change to "mkdocs" in new structure
+      MKDOCS_MODEL_YML: "mkdocs-files.yml"  # contains list of model Markdown files
+      BASE_MKDOCS_YML: "mkdocs.yml"         # initial MkDocs configuration
+      FULL_MKDOCS_YML: "__mkdocs-full.yml"  # MkDocs configuration combined with model list
+      GIT_USER_NAME: "ci-bot"               # for gh-pages commit
+      GIT_USER_EMAIL: "[email protected]"     # for gh-pages commit
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
+      - name: Checkout spdx-spec
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
         with:
           ref: development/v3.0.1
           path: spdx-spec
           fetch-depth: 0  # Because we will be pushing the gh-pages branch
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
-        with:
-          repository: spdx/spec-parser
-          ref: main
-          path: spec-parser
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
+      - name: Checkout spdx-3-model
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
         with:
           repository: spdx/spdx-3-model
           ref: main  # for v3.0.1-draft, checkout latest from main (development)
           # Once 3.0.1 released, use the following ref to checkout from the release tag
           # ref: 3.0.1  # for v3.0.1 final release
           path: spdx-3-model
+      - name: Checkout spec-parser
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
+        with:
+          repository: spdx/spec-parser
+          ref: main
+          path: spec-parser
       - name: Install pre-requisites for spdx-spec
         run: pip install -r spdx-spec/requirements.txt
       - name: Install pre-requisites for spec-parser
         run: pip install -r spec-parser/requirements.txt
       - name: Build model files
-        run: python3 spec-parser/main.py spdx-3-model/model spdx-spec/docs/model
+        run: python3 spec-parser/main.py spdx-3-model/model $PARSER_OUTPUT_DIR
+      - name: Create directories for model (MkDocs) and RDF files
+        run: |
+          mkdir spdx-spec/docs/rdf
+          mkdir spdx-spec/docs/model
       - name: Copy JSON annotations
-        run: cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/model/jsonld-annotations.ttl
-      - name: Generate JSON schema
+        # Will be redirected from https://spdx.org/rdf/3.0.0/spdx-json-serialize-annotations.ttl
+        # and available at https://spdx.github.io/spdx-spec/v3.0/rdf/jsonld-annotations.ttl
+        run: |
+          cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/rdf/jsonld-annotations.ttl
+          cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/model/jsonld-annotations.ttl
+      - name: Copy JSON-LD context and RDFs
+        # Will be redirected from https://spdx.org/rdf/3.0.0/spdx-context.jsonld, spdx-model.ttl, etc.
+        # and available at https://spdx.github.io/spdx-spec/v3.0/model/spdx-context.jsonld
+        run: |
+          echo "====================="
+          echo "Source: $PARSER_OUTPUT_DIR/$RDF_BASE_DIR"
+          echo "---------------------"
+          ls $PARSER_OUTPUT_DIR/$RDF_BASE_DIR
+          echo "====================="
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-context.jsonld spdx-spec/docs/rdf/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.dot spdx-spec/docs/rdf/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.json-ld spdx-spec/docs/rdf/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.pretty-xml spdx-spec/docs/rdf/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.ttl spdx-spec/docs/rdf/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.xml spdx-spec/docs/rdf/
+          cp spdx-spec/docs/rdf/spdx-model.json-ld spdx-spec/docs/rdf/spdx-model.jsonld
+          echo "====================="
+          echo "Target (after copy): spdx-spec/docs/rdf"
+          echo "---------------------"
+          ls spdx-spec/docs/rdf
+          echo "====================="
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-context.jsonld spdx-spec/docs/model/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.dot spdx-spec/docs/model/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.json-ld spdx-spec/docs/model/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.pretty-xml spdx-spec/docs/model/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.ttl spdx-spec/docs/model/
+          cp $PARSER_OUTPUT_DIR/$RDF_BASE_DIR/spdx-model.xml spdx-spec/docs/model/
+          cp spdx-spec/docs/model/spdx-model.json-ld spdx-spec/docs/model/spdx-model.jsonld
+          echo "====================="
+          echo "Target (after copy): spdx-spec/docs/model"
+          echo "---------------------"
+          ls spdx-spec/docs/model
+          echo "====================="
+      - name: Generate JSON schema -- OLD DIR STRUCTURE
+        # Will be redirected from https://spdx.org/schema/3.0.0/spdx-json-schema.json
+        # and available at https://spdx.github.io/spdx-spec/v3.0/model/schema.json
         run: |
           shacl2code generate \
-            --input spdx-spec/docs/model/spdx-model.ttl \
-            --input spdx-spec/docs/model/jsonld-annotations.ttl \
-            --context-url spdx-spec/docs/model/spdx-context.jsonld https://spdx.org/rdf/3.0.1/spdx-context.jsonld \
+            --input spdx-spec/docs/rdf/spdx-model.ttl \
+            --input spdx-spec/docs/rdf/jsonld-annotations.ttl \
+            --context-url spdx-spec/docs/rdf/spdx-context.jsonld https://spdx.org/rdf/3.0.1/spdx-context.jsonld \
             jsonschema \
-            --output spdx-spec/docs/model/schema.json
-      - name: Set git identity
+            --output spdx-spec/docs/rdf/schema.json
+          cp spdx-spec/docs/rdf/schema.json spdx-spec/docs/model/schema.json
+      - name: Copy model files and file list for MkDocs
+        # Will be available at https://spdx.github.io/spdx-spec/v3.0/model/*
+        run: |
+          cp -R $PARSER_OUTPUT_DIR/$MKDOCS_BASE_DIR/* spdx-spec/docs/model
+          cp $PARSER_OUTPUT_DIR/$MKDOCS_MODEL_YML spdx-spec
+      - name: Set Git identity
         working-directory: spdx-spec
-        run: git config user.name ci-bot; git config user.email [email protected]
+        run: git config user.name $GIT_USER_NAME; git config user.email $GIT_USER_EMAIL
       - name: Sync gh-pages
         working-directory: spdx-spec
         run: git checkout gh-pages && git pull && git checkout development/v3.0.1
-      - name: Build docs and set aliases
+      - name: Build complete MkDocs configuration
         working-directory: spdx-spec
         run: |
-          MODEL_YML="docs/model/mkdocs-files.yml"
-          BASE_MKDOCS_YML="mkdocs.yml"
-          FINAL_MKDOCS_YML="mkdocs-final.yml"
-          echo "Build $FINAL_MKDOCS_YML from $BASE_MKDOCS_YML and $MODEL_YML"
-          sed -e "\|- model.*#.*[MODEL_PLACEHOLDER].*|{
-              r $MODEL_YML
+          echo "Build $FULL_MKDOCS_YML from $BASE_MKDOCS_YML and $MKDOCS_MODEL_YML"
+          sed -e "\|- model.*#.*__MODEL_PLACEHOLDER__.*|{
+              r $MKDOCS_MODEL_YML
               a\\
 
               d
-          }" "$BASE_MKDOCS_YML" > "$FINAL_MKDOCS_YML"
-          echo "====================="
-          echo "Start mike deploy"
+          }" "$BASE_MKDOCS_YML" > "$FULL_MKDOCS_YML"
+          sed -i "/__MODEL_PLACEHOLDER__/d" "$FULL_MKDOCS_YML"
           echo "====================="
-          mike deploy --update-aliase --config-file "$FINAL_MKDOCS_YML" --branch gh-pages --push v3.0.1-draft
-        # Once 3.0.1 released, use the following command to deploy 3.0.1 as latest version
-        # mike deploy --update-aliase --config-file "$FINAL_MKDOCS_YML" --branch gh-pages --push v3.0.1 latest
-#      - name: Set default version
-#        working-directory: spdx-spec
-#        run: mike set-default v3.0
-        # Once 3.0.1 released, use the following command to set 3.0.1 as default version
-        # run: mike set-default v3.0.1
+          echo "$FULL_MKDOCS_YML"
+          echo "---------------------"
+          cat "$FULL_MKDOCS_YML"
+          echo "[End of File]"
+      - name: Deploy and set aliases
+        working-directory: spdx-spec
+        run: |
+          mike delete --config-file "$FULL_MKDOCS_YML" --branch gh-pages --push --allow-empty v3.0 || true
+          mike delete --config-file "$FULL_MKDOCS_YML" --branch gh-pages --push --allow-empty v3.0.1-draft || true
+          mike deploy --update-aliase --config-file "$FULL_MKDOCS_YML" --branch gh-pages --push v3.0.1 v3.0.1-draft v3-draft v3.0-RC1 v3.0-RC2 v3.0 latest
+          mike set-default --config-file "$FULL_MKDOCS_YML" --branch gh-pages --push v3.0.1
+        # Explanations of the above mike steps:
+        # 1) delete existing v3.0 alias/deployment, if exists
+        # 2) delete existing v3.0.1-draft alias/deployment, if exists
+        # 3) deploy v3.0.1, with aliases (v3.0.1-draft, v3-draft, v3.0-RC1, v3.0-RC2, v3.0 latest)
+        # 4) set default version to v3.0.1
+      # A script for additional redirections will be run after this point
+      # See https://github.com/spdx/spdx-spec/issues/1069

.github/workflows/validate_pull_request.yml

@@ -6,44 +6,13 @@ jobs:
     runs-on: ubuntu-latest
     container: python:3
     steps:
-      - name: Checkout spdx-spec
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
-        with:
-          path: spdx-spec
-          fetch-depth: 1
-      - name: Checkout spdx-3-model
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
-        with:
-          repository: spdx/spdx-3-model
-          ref: main
-          path: spdx-3-model
-      - name: Checkout spec-parser
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
-        with:
-          repository: spdx/spec-parser
-          ref: main
-          path: spec-parser
-      - name: Install pre-requisites for spdx-spec
-        run: pip install -r spdx-spec/requirements.txt
-      - name: Install pre-requisites for spec-parser
-        run: pip install -r spec-parser/requirements.txt
-      - name: Build model files
-        run: python3 spec-parser/main.py spdx-3-model/model spdx-spec/docs/model
-      - name: Build the site to validate
-        working-directory: spdx-spec
-        run: |
-          MODEL_YML="docs/model/mkdocs-files.yml"
-          BASE_MKDOCS_YML="mkdocs.yml"
-          FINAL_MKDOCS_YML="mkdocs-final.yml"
-          echo "Build $FINAL_MKDOCS_YML from $BASE_MKDOCS_YML and $MODEL_YML"
-          sed -e "\|- model.*#.*[MODEL_PLACEHOLDER].*|{
-              r $MODEL_YML
-              a\\
-
-              d
-          }" "$BASE_MKDOCS_YML" > "$FINAL_MKDOCS_YML"
-          echo "====================="
-          echo "Start mkdocs build"
-          echo "====================="
-          ENABLE_PDF_EXPORT=0 mkdocs build --clean --config-file "$FINAL_MKDOCS_YML" --verbose
-        # Build without the time-consuming PDF export
+    - name: Checkout spdx-spec
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  #v4.1.7
+      with:
+        fetch-depth: 1
+    - name: Install pre-requisites
+      run: pip install -r requirements.txt
+    - name: Build the site to validate (use mkdocs.yml, no model files, no PDF export)
+      run: ENABLE_PDF_EXPORT=0 mkdocs build --clean --verbose
+      # To validate the PR from within spdx-spec repo, build using mkdocs.yml
+      # (no model files) and without the time-consuming PDF export

.gitignore

@@ -11,7 +11,7 @@ _book/
 dist/
 site/
 
-# Ignore all hidden files/dirs except .gitignore
+# Ignore all hidden files/dirs except .gitignore and .github
 .*
 !/.gitignore
 !/.github
@@ -28,3 +28,9 @@ node_modules/
 Thumbs.db
 Thumbs.db:encryptable
 *.lnk
+
+# Build temporary directory
+.env
+logs/
+__pycache__/
+__mkdocs-full.yml