Automate functional test in the pipeline (#1634)

Problem: As a user, I want to automate functional tests in the pipeline

Solution: To run functional tests in the pipeline, we introduce a new github workflow that sets up the environment to run tests, and upload results

Testing: Manual testing by pushing to the branch

.github/workflows/functional.yml

@@ -0,0 +1,124 @@
+name: Functional Testing
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.ref_name }}-functional
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  functional-tests:
+    name: Gateway Functional Tests
+    runs-on: ubuntu-22.04
+    strategy:
+        matrix:
+          k8s-version: ["1.23.17", "latest"]
+          nginx-image: [nginx, nginx-plus]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Setup Golang Environment
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: stable
+
+      - name: Set GOPATH
+        run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+
+
+      - name: NGF Docker meta
+        id: ngf-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ matrix.nginx-image }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Build binary
+        uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
+        with:
+          version: latest
+          args: build --snapshot --clean
+
+      - name: Build NGF Docker Image
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.ngf-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=ngf
+          cache-to: type=gha,scope=ngf,mode=max
+          pull: true
+          target: goreleaser
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        with:
+          file: build/Dockerfile${{ matrix.nginx-image == 'nginx' && '.nginx' || '' }}${{ matrix.nginx-image == 'nginx-plus' && '.nginxplus' || ''}}
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=${{ matrix.nginx-image }}
+          cache-to: type=gha,scope=${{ matrix.nginx-image }},mode=max
+          pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
+            BUILD_AGENT=gha
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          k8s_version=${{ matrix.k8s-version }}
+          make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(matrix.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
+          echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
+
+      - name: Setup functional tests
+        id: setup
+        run: |
+          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
+          ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          make load-images${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
+        working-directory: ./tests
+
+      - name: Run functional tests
+        run: |
+          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
+          ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          make test${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
+        working-directory: ./tests

tests/Makefile

@@ -106,6 +106,10 @@ test: ## Runs the functional tests on your default k8s cluster
 		--pull-policy=$(PULL_POLICY) --k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) \
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
+.PHONY: test-with-plus
+test-with-plus: ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster
+	make test PLUS_ENABLED=true
+
 .PHONY: cleanup-gcp
 cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
 

tests/README.md

@@ -15,7 +15,7 @@ internal LoadBalancer service which will receive the test traffic.
 Directory structure is as follows:
 
 - `framework`: contains utility functions for running the tests
-- `results`: contains the results files
+- `results`: contains the results files for the NFR tests
 - `scripts`: contain scripts used to set up the environment and run the tests
 - `suite`: contains the test files
 
@@ -63,6 +63,7 @@ start-longevity-test           Start the longevity test to run for 4 days in GKE
 stop-longevity-test            Stops the longevity test and collects results
 sync-files-to-vm               Syncs your local NGF files with the NGF repo on the VM
 test                           Runs the functional tests on your default k8s cluster
+test-with-plus                 Runs the functional tests for NGF with NGINX Plus on your default k8s cluster
 ```
 
 **Note:** The following variables are configurable when running the below `make` commands: