misc-sync-templates.yml

name: Synchronize templates


# This job is used to keep the repository templates up-to-date.
# The code of the templates exist inside the monorepo, and upon releases we synchronize the repositories:
# - https://github.com/paritytech/polkadot-sdk-minimal-template
# - https://github.com/paritytech/polkadot-sdk-parachain-template
# - https://github.com/paritytech/polkadot-sdk-solochain-template
#
# The job moves the template code out of the monorepo,
# replaces any references to the monorepo workspace using psvm and toml-cli,
# checks that it builds successfully,
# and commits and pushes the result to each respective repository.
# If the build fails, a PR is created instead for manual inspection.


on:
  # A manual dispatch for now - automatic on releases later.
  workflow_dispatch:
    inputs:
      stable_release_branch:
        description: 'Stable release branch, e.g. stable2407'
        required: true
      debug:
        description: Enable runner debug logging
        required: false
        default: false
      patch:
        description: 'Patch number of the stable release we want to sync with'
        required: false
        default: ""

jobs:
  prepare-chain-spec-artifacts:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - template: minimal
            package_name: 'minimal-template-runtime'
            runtime_path: './templates/minimal/runtime'
            runtime_wasm_path: minimal-template-runtime/minimal_template_runtime.compact.compressed.wasm
            relay_chain: 'dev'
          - template: parachain
            package_name: 'parachain-template-runtime'
            runtime_path: './templates/parachain/runtime'
            runtime_wasm_path: parachain-template-runtime/parachain_template_runtime.compact.compressed.wasm
            relay_chain: 'rococo-local'
    steps:
        - uses: actions/checkout@v4
          with:
            ref: "${{ github.event.inputs.stable_release_branch }}"

        - name: Setup build environment
          run: |
            sudo apt-get update
            sudo apt-get install -y protobuf-compiler
            cargo install --git https://github.com/chevdor/srtool-cli --locked
            cargo install --path substrate/bin/utils/chain-spec-builder --locked
            srtool pull
            
        - name: Build runtime and generate chain spec
          run: |
            # Prepare directories
            sudo mkdir -p ${{ matrix.runtime_path }}/target
            sudo chmod -R 777 ${{ matrix.runtime_path }}/target
            
            # Build runtime
            srtool build --package ${{ matrix.package_name }} --runtime-dir ${{ matrix.runtime_path }} --root
            
            # Generate chain spec
            # Note that para-id is set to 1000 for both minimal/parachain templates. 
            # `parachain-runtime` is hardcoded to use this parachain id. 
            # `minimal` template isn't using it, but when started with Omni Node, this para id is required (any number can do it, so setting it to 1000 for convenience).
            chain-spec-builder -c dev_chain_spec.json create \
              --relay-chain "${{ matrix.relay_chain }}" \
              --para-id 1000 \
              --runtime "${{ matrix.runtime_path }}/target/srtool/release/wbuild/${{ matrix.runtime_wasm_path }}" \
              named-preset development

        - name: Prepare upload directory
          run: |
            mkdir -p artifacts-${{ matrix.template }}
            cp dev_chain_spec.json artifacts-${{ matrix.template }}/dev_chain_spec.json

        - name: Upload template directory
          uses: actions/upload-artifact@v4
          with:
            name: artifacts-${{ matrix.template }}
            path: artifacts-${{ matrix.template }}/dev_chain_spec.json

  sync-templates:
    needs: prepare-chain-spec-artifacts
    runs-on: ubuntu-latest
    environment: master
    strategy:
      fail-fast: false
      matrix:
        template: ["minimal", "solochain", "parachain"]
    env:
      template-path: "polkadot-sdk-${{ matrix.template }}-template"
    steps:

      # 1. Prerequisites.

      - name: Configure git identity
        run: |
          git config --global user.name "Template Bot"
          git config --global user.email "163342540+paritytech-polkadotsdk-templatebot[bot]@users.noreply.github.com"
      - uses: actions/checkout@v4
        with:
          path: polkadot-sdk
          ref: "${{ github.event.inputs.stable_release_branch }}"
      - name: Download template artifacts
        uses: actions/download-artifact@v4
        with:
          name: artifacts-${{ matrix.template }}
          path: templates/${{ matrix.template }}/
        if: matrix.template != 'solochain'
      - name: Generate a token for the template repository
        id: app_token
        uses: actions/create-github-app-token@v1.9.3
        with:
          owner: "paritytech"
          repositories: "polkadot-sdk-${{ matrix.template }}-template"
          app-id: ${{ secrets.TEMPLATE_APP_ID }}
          private-key: ${{ secrets.TEMPLATE_APP_KEY }}
      - uses: actions/checkout@v4
        with:
          repository: "paritytech/polkadot-sdk-${{ matrix.template }}-template"
          path: "${{ env.template-path }}"
          token: ${{ steps.app_token.outputs.token }}
      - name: Install toml-cli
        run: cargo install --git https://github.com/gnprice/toml-cli --rev ea69e9d2ca4f0f858110dc7a5ae28bcb918c07fb # v0.2.3
      - name: Install Polkadot SDK Version Manager
        run: cargo install --git https://github.com/paritytech/psvm psvm
      - name: Rust compilation prerequisites
        run: |
          sudo apt update
          sudo apt install -y \
            protobuf-compiler
          rustup target add wasm32-unknown-unknown
          rustup component add rustfmt clippy rust-src

      # 2. Yanking the template out of the monorepo workspace.

      - name: Replace dev-dependencies path references with workspace references
        run: find . -type f -name 'Cargo.toml' -exec sed -i'' -E "s/path = \"\.\.\/.*\"/workspace = true/g" {} \; 
        working-directory: polkadot-sdk/templates/${{ matrix.template }}/
      - name: Create a new workspace Cargo.toml
        run: |
          # This replaces the existing Cargo.toml for parachain-template,
          # corresponding to the `parachain-template-docs` crate, so no need
          # to delete that `Cargo.toml` after copying the `polkadot-sdk/templates/parachain/*`
          # to the `polkadot-sdk-parachain-template` repo.
          cat << EOF > Cargo.toml
          [workspace.package]
          license = "MIT-0"
          authors = ["Parity Technologies <admin@parity.io>"]
          homepage = "https://paritytech.github.io/polkadot-sdk/"

          [workspace]
          EOF

          [ ${{ matrix.template }} != "solochain" ] && echo "# Leave out the node compilation from regular template usage." \
            && echo "default-members = [\"pallets/template\", \"runtime\"]" >> Cargo.toml
          [ ${{ matrix.template }} == "solochain" ] && echo "# The node isn't yet replaceable by Omni Node."
          cat << EOF >> Cargo.toml
          members = [
              "node",
              "pallets/template",
              "runtime",
          ]
          resolver = "2"

          [workspace.dependencies]
          EOF

          echo "$(toml get -r ./runtime/Cargo.toml 'package.name') = { path = \"./runtime\", default-features = false }" >> Cargo.toml
          echo "$(toml get -r ./pallets/template/Cargo.toml 'package.name') = { path = \"./pallets/template\", default-features = false }" >> Cargo.toml
        shell: bash
        working-directory: polkadot-sdk/templates/${{ matrix.template }}/
      - name: Update workspace configuration
        run: |
          set -euo pipefail
          # toml-cli has no overwrite functionality yet, so we use temporary files.
          # We cannot pipe the output straight to the same file while the CLI still reads and processes it.

          toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.repository' "https://github.com/paritytech/polkadot-sdk-${{ matrix.template }}-template.git" > Cargo.temp
          mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml

          toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.edition' "$(toml get --raw Cargo.toml 'workspace.package.edition')" > Cargo.temp
          mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
        working-directory: polkadot-sdk

      - name: Print the result Cargo.tomls for debugging
        if: ${{ github.event.inputs.debug }}
        run: find . -type f -name 'Cargo.toml' -exec cat {} \;
        working-directory: polkadot-sdk/templates/${{ matrix.template }}/

      - name: Clean the destination repository
        run: rm -rf ./*
        working-directory: "${{ env.template-path }}"
      - name: Copy over the new changes
        run: |
          cp -r polkadot-sdk/templates/${{ matrix.template }}/* "${{ env.template-path }}/"
      - name: Remove unnecessary files from parachain template
        if: ${{ matrix.template == 'parachain' }}
        run: |
          rm -f "${{ env.template-path }}/README.docify.md"
          rm -f "${{ env.template-path }}/src/lib.rs"

      - name: Run psvm on monorepo workspace dependencies
        run: |
          patch_input="${{ github.event.inputs.patch }}"
          if [[ -n "$patch_input" ]]; then
            patch="-$patch_input"
          else
            patch=""
          fi
          psvm -o -v "${{ github.event.inputs.stable_release_branch }}$patch" -p ./Cargo.toml
        working-directory: polkadot-sdk/
      - name: Copy over required workspace dependencies
        run: |
          set +e
          # If a workspace dependency is required..
          while cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`'; do
            # Get its name..
            missing_dep=$(cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`' | sed -E 's/(.*)`dependency.(.*)` was not found in `workspace.dependencies`/\2/')
            # And copy the dependency from the monorepo.
            toml get ../polkadot-sdk/Cargo.toml 'workspace.dependencies' --output-toml | grep "^${missing_dep} = " >> Cargo.toml
          done;
        working-directory: "${{ env.template-path }}"

      - name: Print the result Cargo.tomls for debugging after copying required workspace dependencies
        if: ${{ github.event.inputs.debug }}
        run: find . -type f -name 'Cargo.toml' -exec cat {} \;
        working-directory: polkadot-sdk/templates/${{ matrix.template }}/


      # 3. Verify the build. Push the changes or create a PR.

      # We've run into out-of-disk error when compiling in the next step, so we free up some space this way.
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # 1.3.1
        with:
          android: true # This alone is a 12 GB save.
          # We disable the rest because it caused some problems. (they're enabled by default)
          # The Android removal is enough.
          dotnet: false
          haskell: false
          large-packages: false
          swap-storage: false

      - name: Check if it compiles
        id: check-compilation
        run: cargo check && cargo test
        working-directory: "${{ env.template-path }}"
        timeout-minutes: 90
      - name: Create PR on failure
        if: failure() && steps.check-compilation.outcome == 'failure'
        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v5
        with:
          path: "${{ env.template-path }}"
          token: ${{ steps.app_token.outputs.token }}
          add-paths: |
            ./*
          title: "[Don't merge] Update the ${{ matrix.template }} template to ${{ github.event.inputs.stable_release_branch }}"
          body: "The template has NOT been successfully built and needs to be inspected."
          branch: "update-template/${{ github.event.inputs.stable_release_branch }}"
      - name: Create PR on success
        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v5
        with:
          path: "${{ env.template-path }}"
          token: ${{ steps.app_token.outputs.token }}
          add-paths: |
            ./*
          title: "Update the ${{ matrix.template }} template to ${{ github.event.inputs.stable_release_branch }}"
          body: "This synchronizes the template to the ${{ github.event.inputs.stable_release_branch }} branch."
          branch: "update-template/${{ github.event.inputs.stable_release_branch }}"