e2e-test-sign.yml

# Copyright The Notary Project Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: e2e-test-sign

on:
  push:
  pull_request:

env:
  E2E_KEY: /home/runner/.config/notation/localkeys/e2e-test.key
  E2E_CERT: /home/runner/.config/notation/localkeys/e2e-test.crt

jobs:
  e2e-test-sign:
    runs-on: ubuntu-latest
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Build and push to local registry
        id: image1
        uses: docker/build-push-action@v6
        with:
          context: ./tests/e2e
          push: true
          tags: localhost:5000/image1:latest
      - name: Build and push to local registry
        id: image2
        uses: docker/build-push-action@v6
        with:
          context: ./tests/e2e
          push: true
          tags: localhost:5000/image2:latest
      - name: Retrieve digest
        run: |
          echo "target_artifact_reference=localhost:5000/image1@${{ steps.image1.outputs.digest }}" >> "$GITHUB_ENV"
          echo "target_artifact_reference2=localhost:5000/image2@${{ steps.image2.outputs.digest }}" >> "$GITHUB_ENV"
      
      # Setting up Notation CLI on the runner
      - name: Setup Notation
        uses: ./setup
      
      # Generate test
      - name: Notation generate-test
        run: |
          notation cert generate-test "e2e-test"

      # E2E test cases on Notation Sign
      - name: Sign artifact using notation plugin
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      
      - name: Sign multiple artifacts using notation plugin
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: |-
            ${{ env.target_artifact_reference }}
            ${{ env.target_artifact_reference2 }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}

      - name: Sign artifact with multiple plugin_config
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
            key=val
            foo=bar
      
      - name: Sign artifact with timestamping
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          timestamp_url: http://timestamp.digicert.com
          timestamp_root_cert: ./tests/e2e/sign/tsaRootCert/DigiCertTSARootSHA384.cer
      
      - name: Sign artifact with force_referrers_tag set to true
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          force_referrers_tag: 'true'
      
      - name: Sign artifact with force_referrers_tag set to false
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          force_referrers_tag: 'false'

      - name: Sign artifact with allow_referrers_api set to true
        uses: ./sign
        env:
          NOTATION_EXPERIMENTAL: 1
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          allow_referrers_api: 'true'
      
      - name: Sign with force_referrers_tag set to invalid value
        continue-on-error: true
        id: invalid-force_referrers_tag
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: invalid-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          force_referrers_tag: 'invalid'
      - name: 'Should Fail: Sign with force_referrers_tag set to invalid value'
        if: steps.invalid-force_referrers_tag.outcome != 'failure'
        run: |
            echo "Sign with force_referrers_tag set to invalid value should fail, but succeeded."
            exit 1
      
      - name: Sign artifact with invalid plugin name
        continue-on-error: true
        id: invalid-plugin-name
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: invalid-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      - name: 'Should Fail: Sign artifact with invalid plugin name'
        if: steps.invalid-plugin-name.outcome != 'failure'
        run: |
            echo "Sign artifact with invalid plugin name should fail, but succeeded."
            exit 1

      - name: Sign artifact with invalid plugin url
        continue-on-error: true
        id: invalid-plugin-url
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: test
          plugin_url: invalid-url
          plugin_checksum: test
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      - name: 'Should Fail: Sign artifact with invalid plugin url'
        if: steps.invalid-plugin-url.outcome != 'failure'
        run: |
            echo "Sign artifact with invalid plugin url should fail, but succeeded."
            exit 1
      
      - name: Sign artifact missing plugin checksum
        continue-on-error: true
        id: missing-plugin-checksum
        uses: ./sign
        with:
          plugin_name: test
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      - name: 'Should Fail: Sign artifact missing plugin checksum'
        if: steps.missing-plugin-checksum.outcome != 'failure'
        run: |
            echo "Sign artifact missing plugin checksum should fail, but succeeded."
            exit 1
      
      - name: Sign artifact with invalid plugin checksum
        continue-on-error: true
        id: invalid-plugin-checksum
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: test
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: abcdef
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      - name: 'Should Fail: Sign artifact with invalid plugin checksum'
        if: steps.invalid-plugin-checksum.outcome != 'failure'
        run: |
            echo "Sign artifact with invalid plugin checksum should fail, but succeeded."
            exit 1

      - name: Sign artifact missing key id
        continue-on-error: true
        id: missing-key-id
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: test
          plugin_url: https://test
          plugin_checksum: abcdef
          target_artifact_reference: test
      - name: 'Should Fail: Sign artifact missing key id'
        if: steps.missing-key-id.outcome != 'failure'
        run: |
            echo "Sign artifact missing key id should fail, but succeeded."
            exit 1

      - name: Sign artifact missing target artifact reference
        continue-on-error: true
        id: missing-artifact-reference
        uses: ./sign
        env:
          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
        with:
          plugin_name: test
          plugin_url: https://test
          plugin_checksum: abcdef
          key_id: test
      - name: 'Should Fail: Sign artifact missing target artifact reference'
        if: steps.missing-artifact-reference.outcome != 'failure'
        run: |
            echo "Sign artifact missing target artifact reference should fail, but succeeded."
            exit 1
      
      - name: Sign artifact with timestamp url but no timestamp root cert
        continue-on-error: true
        id: missing-timestamp-root-cert
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          timestamp_url: http://timestamp.digicert.com
      - name: 'Should Fail: Sign artifact with timestamp url but no timestamp root cert'
        if: steps.missing-timestamp-root-cert.outcome != 'failure'
        run: |
              echo "Sign artifact with timestamp url but no timestamp root cert should fail, but succeeded."
              exit 1
      
      - name: Sign artifact with timestamp root cert but no timestamp url
        continue-on-error: true
        id: missing-timestamp-url
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          timestamp_root_cert: ./tests/e2e/sign/tsaRootCert/DigiCertTSARootSHA384.cer
      - name: 'Should Fail: Sign artifact with timestamp root cert but no timestamp url'
        if: steps.missing-timestamp-url.outcome != 'failure'
        run: |
              echo "Sign artifact with timestamp root cert but no timestamp url should fail, but succeeded."
              exit 1
      
      - name: clean up plugin
        run: |
          notation plugin uninstall e2e-test-plugin --yes
      - name: Setup older Notation
        uses: ./setup
        with:
          version: "1.1.1"
      - name: Sign with older Notation
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
      
      - name: Sign with older Notation and allow_referrers_api set to true
        uses: ./sign
        env:
          NOTATION_EXPERIMENTAL: 1
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          allow_referrers_api: 'true'
      
      - name: Sign artifact with timestamping using older Notation
        continue-on-error: true
        id: timestamping-with-older-notation
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          timestamp_url: http://timestamp.digicert.com
          timestamp_root_cert: ./tests/e2e/sign/tsaRootCert/DigiCertTSARootSHA384.cer
      - name: 'Should Fail: Sign artifact with timestamping using older Notation'
        if: steps.timestamping-with-older-notation.outcome != 'failure'
        run: |
              echo "Sign artifact with timestamping using older Notation should fail, but succeeded."
              exit 1

      - name: Sign artifact with force_referrers_tag set using older Notation
        continue-on-error: true
        id: force_referrers_tag-set-with-older-notation
        uses: ./sign
        with:
          plugin_name: e2e-test-plugin
          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
          key_id: ${{ env.E2E_CERT }}
          target_artifact_reference: ${{ env.target_artifact_reference }}
          signature_format: cose
          plugin_config: |-
            keyFile=${{ env.E2E_KEY }}
          force_referrers_tag: 'false'
      - name: 'Should Fail: Sign artifact with force_referrers_tag set using older Notation'
        if: steps.force_referrers_tag-set-with-older-notation.outcome != 'failure'
        run: |
              echo "Sign artifact with force_referrers_tag set using older Notation should fail, but succeeded."
              exit 1