NGINX Plus R33 support (#2760)

Adding support for NGINX Plus R33. The major change with this release is that NGINX Plus now requires a JWT in order to run. A user must create a Secret with this JWT and supply the secret name to NGF when installing. A user can also create client SSL and CA Secrets for NIM connections. All of these Secrets are mounted to the nginx container.

Because of the new usage reporting method, the old usage reporting method has been removed and CLI arguments have been altered. Since this release is a breaking change for N+ users, the choice was made to remove the unused usage reporting flags instead of deprecating them.

Updated documentation to describe this process, while also cleaning up the JWT docker registry process for N+.

.github/workflows/ci.yml

@@ -238,6 +238,7 @@ jobs:
     with:
       image: ${{ matrix.image }}
       k8s-version: ${{ matrix.k8s-version }}
+    secrets: inherit
     permissions:
       contents: read
 
@@ -259,6 +260,7 @@ jobs:
       image: ${{ matrix.image }}
       k8s-version: ${{ matrix.k8s-version }}
       enable-experimental: ${{ matrix.enable-experimental }}
+    secrets: inherit
     permissions:
       contents: write
 

.github/workflows/conformance.yml

@@ -135,6 +135,12 @@ jobs:
           kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
           kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
 
+      - name: Setup license file for plus
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Setup conformance tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric

.github/workflows/functional.yml

@@ -100,6 +100,12 @@ jobs:
             NGINX_CONF_DIR=internal/mode/static/nginx/conf
             BUILD_AGENT=gha
 
+      - name: Setup license file for plus
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Install cloud-provider-kind
         run: |
           CLOUD_PROVIDER_KIND_VERSION=v0.4.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind

.github/workflows/helm.yml

@@ -98,6 +98,15 @@ jobs:
           kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
           kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
           kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
+          kubectl create namespace nginx-gateway
+
+      - name: Create plus secret
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: |
+          echo "${PLUS_LICENSE}" > license.jwt
+          kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
@@ -110,7 +119,7 @@ jobs:
 
       - name: Install Chart
         run: |
-          ct install --config .ct.yaml --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
+          ct install --config .ct.yaml --namespace nginx-gateway --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
           --set=nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx${{ inputs.image == 'plus' && '-plus' || ''}} \
           --set=nginx.plus=${{ inputs.image == 'plus' }} \
           --set=nginx.image.tag=${{ steps.nginx-meta.outputs.version }} \
@@ -143,10 +152,14 @@ jobs:
           kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
           kubectl create namespace nginx-gateway
 
-      - name: Create k8s secret
+      - name: Create plus secrets
         if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
         run: |
+          echo "${PLUS_LICENSE}" > license.jwt
           kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway
+          kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0

.github/workflows/nfr.yml

@@ -111,6 +111,12 @@ jobs:
           echo "GKE_NUM_NODES=12" >> vars.env
           echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
 
+      - name: Setup license file for plus
+        if: matrix.type == 'plus'
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Create GKE cluster
         working-directory: ./tests
         run: make create-gke-cluster CI=true

.gitignore

@@ -46,6 +46,9 @@ internal/mode/static/nginx/modules/coverage
 *.crt
 *.key
 
+# JWT files
+*.jwt
+
 # Dotenv files
 **/*.env
 

.yamllint.yaml

@@ -3,6 +3,7 @@ ignore:
   - charts/nginx-gateway-fabric/templates
   - config/crd/bases/
   - deploy/crds.yaml
+  - deploy/*nginx-plus
   - site/static
 
 rules:

Makefile

@@ -1,13 +1,12 @@
 # variables that should not be overridden by the user
 VERSION = edge
-SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+SELF_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 CHART_DIR = $(SELF_DIR)charts/nginx-gateway-fabric
 NGINX_CONF_DIR = internal/mode/static/nginx/conf
 NJS_DIR = internal/mode/static/nginx/modules/src
 KIND_CONFIG_FILE = $(SELF_DIR)config/cluster/kind-cluster.yaml
-NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
-BUILD_AGENT=local
-PLUS_ENABLED ?= false
+NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=$(SELF_DIR)nginx-repo.crt --secret id=nginx-repo.key,src=$(SELF_DIR)nginx-repo.key
+BUILD_AGENT = local
 
 PROD_TELEMETRY_ENDPOINT = oss.edge.df.f5.com:443
 # the telemetry related variables below are also configured in goreleaser.yml
@@ -49,6 +48,8 @@ TARGET ?= local## The target of the build. Possible values: local and container
 OUT_DIR ?= build/out## The folder where the binary will be stored
 GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
+PLUS_ENABLED ?= false
+PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt
 override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
 
 .DEFAULT_GOAL := help
@@ -227,7 +228,9 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind
 
 .PHONY: helm-install-local-with-plus
 helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
-	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
+	kubectl create namespace nginx-gateway || true
+	kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true
+	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
 
 # Debug Targets
 .PHONY: debug-build

build/Dockerfile.nginx

@@ -6,7 +6,7 @@ ARG NGINX_CONF_DIR
 ARG BUILD_AGENT
 
 RUN apk add --no-cache libcap \
-    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && mkdir -p /usr/lib/nginx/modules \
 	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
 	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
@@ -18,7 +18,7 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
-RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
+RUN chown -R 101:1001 /etc/nginx /var/cache/nginx
 
 LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
 

build/Dockerfile.nginxplus

@@ -7,7 +7,7 @@ ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.p
 
 FROM alpine:3.20
 
-ARG NGINX_PLUS_VERSION=R32
+ARG NGINX_PLUS_VERSION=R33
 ARG NJS_DIR
 ARG NGINX_CONF_DIR
 ARG BUILD_AGENT
@@ -19,7 +19,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
     && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
     && printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
     && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
-    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && mkdir -p /usr/lib/nginx/modules \
 	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
 	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \

charts/nginx-gateway-fabric/README.md

@@ -268,10 +268,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
-| `nginx.usage.clusterName` | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | string | `""` |
-| `nginx.usage.insecureSkipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
-| `nginx.usage.secretName` | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | string | `""` |
-| `nginx.usage.serverURL` | The base server URL of the NGINX Plus usage reporting server. | string | `""` |
+| `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
+| `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
+| `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |
+| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
+| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
+| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
 | `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
 | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |
 | `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` |

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -18,7 +18,7 @@ rules:
   - get
   - list
   - watch
-{{- if .Values.nginxGateway.productTelemetry.enable }}
+{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
 - apiGroups:
   - ""
   resources:

charts/nginx-gateway-fabric/templates/configmap.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-includes-bootstrap
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "nginx-gateway.labels" . | nindent 4 }}
+data:
+  main.conf: |
+    {{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }}
+    error_log stderr {{ .Values.nginx.config.logging.errorLevel }};
+    {{ else }}
+    error_log stderr info;
+    {{- end }}
+  {{- if .Values.nginx.plus }}
+  mgmt.conf: |
+    mgmt {
+        {{- if .Values.nginx.usage.endpoint }}
+        usage_report endpoint={{ .Values.nginx.usage.endpoint }};
+        {{- end }}
+        {{- if .Values.nginx.usage.skipVerify }}
+        ssl_verify off;
+        {{- end }}
+        {{- if .Values.nginx.usage.caSecretName }}
+        ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt;
+        {{- end }}
+        {{- if .Values.nginx.usage.clientSSLSecretName }}
+        ssl_certificate        /etc/nginx/certs-bootstrap/tls.crt;
+        ssl_certificate_key    /etc/nginx/certs-bootstrap/tls.key;
+        {{- end }}
+        enforce_initial_report off;
+    }
+  {{- end }}

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -42,8 +42,12 @@ spec:
         - copy
         - --source
         - /includes/main.conf
+        {{- if .Values.nginx.plus }}
+        - --source
+        - /includes/mgmt.conf
+        {{- end }}
         - --destination
-        - /etc/nginx/main-includes/main.conf
+        - /etc/nginx/main-includes
         securityContext:
           seccompProfile:
             type: RuntimeDefault
@@ -56,7 +60,7 @@ spec:
           runAsUser: 102
           runAsGroup: 1001
         volumeMounts:
-        - name: nginx-includes-configmap
+        - name: nginx-includes-bootstrap
           mountPath: /includes
         - name: nginx-main-includes
           mountPath: /etc/nginx/main-includes
@@ -69,6 +73,24 @@ spec:
         - --service={{ include "nginx-gateway.fullname" . }}
         {{- if .Values.nginx.plus }}
         - --nginx-plus
+          {{- if .Values.nginx.usage.secretName }}
+        - --usage-report-secret={{ .Values.nginx.usage.secretName }}
+          {{- end }}
+          {{- if .Values.nginx.usage.endpoint }}
+        - --usage-report-endpoint={{ .Values.nginx.usage.endpoint }}
+          {{- end }}
+          {{- if .Values.nginx.usage.resolver }}
+        - --usage-report-resolver={{ .Values.nginx.usage.resolver }}
+          {{- end }}
+          {{- if .Values.nginx.usage.skipVerify }}
+        - --usage-report-skip-verify
+          {{- end }}
+          {{- if .Values.nginx.usage.caSecretName }}
+        - --usage-report-ca-secret={{ .Values.nginx.usage.caSecretName }}
+          {{- end }}
+          {{- if .Values.nginx.usage.clientSSLSecretName }}
+        - --usage-report-client-ssl-secret={{ .Values.nginx.usage.clientSSLSecretName }}
+          {{- end }}
         {{- end }}
         {{- if .Values.metrics.enable }}
         - --metrics-port={{ .Values.metrics.port }}
@@ -94,18 +116,6 @@ spec:
         {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
         - --gateway-api-experimental-features
         {{- end }}
-        {{- if .Values.nginx.usage.secretName }}
-        - --usage-report-secret={{ .Values.nginx.usage.secretName }}
-        {{- end }}
-        {{- if .Values.nginx.usage.serverURL }}
-        - --usage-report-server-url={{ .Values.nginx.usage.serverURL }}
-        {{- end }}
-        {{- if .Values.nginx.usage.clusterName }}
-        - --usage-report-cluster-name={{ .Values.nginx.usage.clusterName }}
-        {{- end }}
-        {{- if .Values.nginx.usage.insecureSkipVerify }}
-        - --usage-report-skip-verify
-        {{- end }}
         {{- if .Values.nginxGateway.snippetsFilters.enable }}
         - --snippets-filters
         {{- end }}
@@ -214,6 +224,19 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-includes
           mountPath: /etc/nginx/includes
+        {{- if .Values.nginx.plus }}
+        - name: nginx-lib
+          mountPath: /var/lib/nginx/state
+          {{- if .Values.nginx.usage.secretName }}
+        - name: nginx-plus-license
+          mountPath: /etc/nginx/license.jwt
+          subPath: license.jwt
+          {{- end }}
+          {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+        - name: nginx-plus-usage-certs
+          mountPath: /etc/nginx/certs-bootstrap/
+          {{- end }}
+        {{- end }}
         {{- with .Values.nginx.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}
         {{- end }}
@@ -257,9 +280,31 @@ spec:
         emptyDir: {}
       - name: nginx-includes
         emptyDir: {}
-      - name: nginx-includes-configmap
+      - name: nginx-includes-bootstrap
         configMap:
-          name: nginx-includes
+          name: nginx-includes-bootstrap
+      {{- if .Values.nginx.plus }}
+      - name: nginx-lib
+        emptyDir: {}
+        {{- if .Values.nginx.usage.secretName }}
+      - name: nginx-plus-license
+        secret:
+          secretName: {{ .Values.nginx.usage.secretName }}
+        {{- end }}
+        {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+      - name: nginx-plus-usage-certs
+        projected:
+          sources:
+          {{- if .Values.nginx.usage.caSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.caSecretName }}
+          {{- end }}
+          {{- if .Values.nginx.usage.clientSSLSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.clientSSLSecretName }}
+          {{- end }}
+        {{- end }}
+      {{- end }}
       {{- with .Values.extraVolumes -}}
       {{ toYaml . | nindent 6 }}
       {{- end }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -33,6 +33,7 @@ volumes:
 - emptyDir
 - secret
 - configMap
+- projected
 users:
 - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }}
 allowedCapabilities: