conformance.yml

name: Conformance Testing

on:
  push:
    branches:
      - main
      - release-*
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+*"
  pull_request:
  schedule:
    - cron: "0 4 * * *" # run every day at 4am UTC

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.ref_name }}-conformance
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  conformance-tests:
    name: Gateway Conformance Tests
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        k8s-version: ["1.23.17", "latest"]
        nginx-image: [nginx, nginx-plus]
        enable-experimental: [true, false]
    permissions:
      contents: write # needed for uploading release artifacts
    steps:
      - name: Checkout Repository
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2

      - name: Setup Golang Environment
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: stable

      - name: Set GOPATH
        run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV

      - name: Docker Buildx
        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0

      - name: NGF Docker meta
        id: ngf-meta
        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
        with:
          images: |
            name=ghcr.io/nginxinc/nginx-gateway-fabric
          tags: |
            type=semver,pattern={{version}}
            type=edge
            type=ref,event=pr
            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}

      - name: NGINX Docker meta
        id: nginx-meta
        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
        with:
          images: |
            name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ matrix.nginx-image }}
          tags: |
            type=semver,pattern={{version}}
            type=edge
            type=ref,event=pr
            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}

      - name: Prepare NGF files
        run: |
          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
          ngf_tag=${{ steps.ngf-meta.outputs.version }}
          make update-ngf-manifest${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
        working-directory: ./conformance

      - name: Build binary
        uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
        with:
          version: latest
          args: build --snapshot --clean
        env:
          TELEMETRY_ENDPOINT: "" # disables sending telemetry
          TELEMETRY_ENDPOINT_INSECURE: "false"

      - name: Build NGF Docker Image
        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
        with:
          file: build/Dockerfile
          tags: ${{ steps.ngf-meta.outputs.tags }}
          context: "."
          target: goreleaser
          load: true
          cache-from: type=gha,scope=ngf
          cache-to: type=gha,scope=ngf,mode=max
          pull: true

      - name: Build NGINX Docker Image
        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
        with:
          file: build/Dockerfile${{ matrix.nginx-image == 'nginx' && '.nginx' || '' }}${{ matrix.nginx-image == 'nginx-plus' && '.nginxplus' || ''}}
          tags: ${{ steps.nginx-meta.outputs.tags }}
          context: "."
          load: true
          cache-from: type=gha,scope=${{ matrix.nginx-image }}
          cache-to: type=gha,scope=${{ matrix.nginx-image }},mode=max
          pull: true
          build-args: |
            NJS_DIR=internal/mode/static/nginx/modules/src
            NGINX_CONF_DIR=internal/mode/static/nginx/conf
            BUILD_AGENT=gha

      - name: Update Go Modules
        if: ${{ github.event_name == 'schedule' }}
        run: make update-go-modules
        working-directory: ./conformance

      - name: Build Test Docker Image
        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
        with:
          file: conformance/tests/Dockerfile
          tags: conformance-test-runner:${{ github.sha }}
          context: "."
          load: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          pull: true

      - name: Deploy Kubernetes
        id: k8s
        run: |
          k8s_version=${{ matrix.k8s-version }}
          make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(matrix.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
          echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
        working-directory: ./conformance

      - name: Wait for release to exist
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          REF=${{ github.ref_name }}
          until docker pull ghcr.io/nginxinc/nginx-gateway-fabric:${REF#v}; do sleep 5; done
          until docker pull ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${REF#v}; do sleep 5; done

      - name: Setup conformance tests
        run: |
          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
          ngf_tag=${{ steps.ngf-meta.outputs.version }}
          if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
          if [ ${{ startsWith(matrix.k8s-version, '1.23') ||  startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi
          if [ ${{ matrix.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
          make install-ngf-local-no-build${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
        working-directory: ./conformance

      - name: Run conformance tests
        run: |
          make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
          core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
          extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
          if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
        working-directory: ./conformance

      - name: Upload profile to release
        if: ${{ matrix.k8s-version == 'latest' && startsWith(github.ref, 'refs/tags/') }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh release upload ${{ github.ref_name }} conformance-profile.yaml
        working-directory: ./conformance