Use staging endpoint for N+ reporting in pipeline (#2885) (#2907)

Problem: Pipeline uses production endpoint for N+ usage reporting,
which should be reserved for production deployments.

Solution: Use staging endpoint in the pipeline. Devs will need to
export the staging endpoint in their environments in order to run
the NFR, functional, and conformance tests locally.

Author: kate-osborn

.github/workflows/conformance.yml

@@ -77,6 +77,7 @@ jobs:
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
           if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
+          export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
           make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
@@ -139,7 +140,7 @@ jobs:
       - name: Setup license file for plus
         if: ${{ inputs.image == 'plus' }}
         env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
         run: echo "${PLUS_LICENSE}" > license.jwt
 
       - name: Setup conformance tests
@@ -148,6 +149,7 @@ jobs:
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
           if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
           if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
+          export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
           make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
           make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests

.github/workflows/functional.yml

@@ -104,7 +104,7 @@ jobs:
       - name: Setup license file for plus
         if: ${{ inputs.image == 'plus' }}
         env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
         run: echo "${PLUS_LICENSE}" > license.jwt
 
       - name: Install cloud-provider-kind
@@ -125,19 +125,22 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
           make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests
 
       - name: Run functional graceful-recovery tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
           make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests
 
       - name: Run functional tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
           make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests

.github/workflows/helm.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Create plus secret
         if: ${{ inputs.image == 'plus' }}
         env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
         run: |
           echo "${PLUS_LICENSE}" > license.jwt
           kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
@@ -155,7 +155,7 @@ jobs:
       - name: Create plus secrets
         if: ${{ inputs.image == 'plus' }}
         env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
         run: |
           echo "${PLUS_LICENSE}" > license.jwt
           kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway

.github/workflows/nfr.yml

@@ -114,7 +114,7 @@ jobs:
       - name: Setup license file for plus
         if: matrix.type == 'plus'
         env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
         run: echo "${PLUS_LICENSE}" > license.jwt
 
       - name: Create GKE cluster

Makefile

@@ -50,6 +50,8 @@ GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd6
 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
 PLUS_ENABLED ?= false
 PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt
+PLUS_USAGE_ENDPOINT ?=## The N+ usage endpoint. For development, please set to the N1 staging endpoint.
+
 override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
 
 .DEFAULT_GOAL := help
@@ -220,17 +222,23 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus
 install-ngf-local-build: build-images load-images helm-install-local ## Install NGF from local build on configured kind cluster.
 
 .PHONY: install-ngf-local-build-with-plus
-install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster.
+install-ngf-local-build-with-plus: check-for-plus-usage-endpoint build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster.
 
 .PHONY: helm-install-local
 helm-install-local: install-gateway-crds ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build.
 	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway $(HELM_PARAMETERS)
 
 .PHONY: helm-install-local-with-plus
-helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
+helm-install-local-with-plus: check-for-plus-usage-endpoint install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
 	kubectl create namespace nginx-gateway || true
 	kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true
-	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
+	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS)
+
+.PHONY: check-for-plus-usage-endpoint
+check-for-plus-usage-endpoint: ## Checks that the PLUS_USAGE_ENDPOINT is set in the environment. This env var is required when deploying or testing with N+.
+ifndef PLUS_USAGE_ENDPOINT
+	$(error PLUS_USAGE_ENDPOINT must be defined in your environment)
+endif
 
 # Debug Targets
 .PHONY: debug-build

docs/developer/quickstart.md

@@ -122,6 +122,7 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
 > Note: You will need a valid NGINX Plus license certificate and key named `nginx-repo.crt` and `nginx-repo.key` in the
 > root of this repo to build the NGINX Plus image.
 > You will also need a valid NGINX Plus JSON Web Token (JWT) to deploy NGF with NGINX Plus. That JWT should be stored in the `license.jwt` file in the root of the `nginx-gateway-fabric/` directory. See the [documentation](https://docs.nginx.com/nginx-gateway-fabric/installation/nginx-plus-jwt/) for instructions on how to download and set up the JWT.
+> Additionally, you need to set the NGINX Plus usage endpoint in your environment. For development and testing, export `PLUS_USAGE_ENDPOINT=<N1 staging endpoint>`.
 
 To build the NGINX Gateway Fabric and NGINX Plus container images from source run the following make command:
 

tests/Makefile

@@ -109,7 +109,7 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
 	./scripts/sync-files-to-vm.sh
 
 .PHONY: nfr-test
-nfr-test: ## Run the NFR tests on a GCP VM
+nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM
 	CI=$(CI) ./scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test
@@ -130,10 +130,10 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \
-		--plus-license-file-name=$(PLUS_LICENSE_FILE)
+		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
 
 .PHONY: test
-test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
+test: check-for-plus-usage-endpoint build-crossplane-image ## Runs the functional tests on your kind k8s cluster
 	kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
 	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
 		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
@@ -143,7 +143,7 @@ test: build-crossplane-image ## Runs the functional tests on your kind k8s clust
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \
-		--plus-license-file-name=$(PLUS_LICENSE_FILE)
+		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
 
 .PHONY: test-with-plus
 test-with-plus: PLUS_ENABLED=true
@@ -177,7 +177,7 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con
 
 .PHONY: generate-static-deployment
 generate-static-deployment:
-	helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml
+	helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) > $(SELF_DIR)config/tests/static-deployment.yaml
 
 # this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job)
 # it overrides the target in the main Makefile when the GW_API_VERSION is set to main

tests/framework/ngf.go

@@ -33,8 +33,9 @@ type InstallationConfig struct {
 	ImageTag             string
 	ImagePullPolicy      string
 	ServiceType          string
-	IsGKEInternalLB      bool
+	PlusUsageEndpoint    string
 	Plus                 bool
+	IsGKEInternalLB      bool
 }
 
 // InstallGatewayAPI installs the specified version of the Gateway API resources.
@@ -78,6 +79,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 	}
 
 	args = append(args, setImageArgs(cfg)...)
+	args = append(args, setPlusUsageEndpointArg(cfg)...)
 	fullArgs := append(args, extraArgs...) //nolint:gocritic
 
 	GinkgoWriter.Printf("Installing NGF with command: helm %v\n", strings.Join(fullArgs, " "))
@@ -226,6 +228,15 @@ func setImageArgs(cfg InstallationConfig) []string {
 	return args
 }
 
+func setPlusUsageEndpointArg(cfg InstallationConfig) []string {
+	var args []string
+	if cfg.Plus && cfg.PlusUsageEndpoint != "" {
+		args = append(args, formatValueSet("nginx.usage.endpoint", cfg.PlusUsageEndpoint)...)
+	}
+
+	return args
+}
+
 func formatValueSet(key, value string) []string {
 	return []string{"--set", fmt.Sprintf("%s=%s", key, value)}
 }

tests/suite/graceful_recovery_test.go

@@ -367,7 +367,7 @@ func checkContainerLogsForErrors(ngfPodName string, checkNginxLogsOnly bool) {
 		Expect(line).ToNot(ContainSubstring("[emerg]"), line)
 		if strings.Contains(line, "[error]") {
 			expectedError1 := "connect() failed (111: Connection refused)"
-			expectedError2 := "product.connect.nginx.com could not be resolved"
+			expectedError2 := "could not be resolved (host not found) during usage report"
 			expectedError3 := "server returned 429"
 			// FIXME(salonichf5) remove this error message check
 			// when https://github.com/nginxinc/nginx-gateway-fabric/issues/2090 is completed.

tests/suite/system_suite_test.go

@@ -62,6 +62,7 @@ var (
 	isGKEInternalLB          = flag.Bool("is-gke-internal-lb", false, "Is the LB service GKE internal only")
 	plusEnabled              = flag.Bool("plus-enabled", false, "Is NGINX Plus enabled")
 	plusLicenseFileName      = flag.String("plus-license-file-name", "", "File name containing the NGINX Plus JWT")
+	plusUsageEndpoint        = flag.String("plus-usage-endpoint", "", "Endpoint for reporting NGINX Plus usage")
 	clusterName              = flag.String("cluster-name", "kind", "Cluster name")
 )
 
@@ -190,12 +191,13 @@ func setup(cfg setupConfig, extraInstallArgs ...string) {
 
 func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framework.InstallationConfig {
 	installCfg := framework.InstallationConfig{
-		ReleaseName:     cfg.releaseName,
-		Namespace:       ngfNamespace,
-		ChartPath:       cfg.chartPath,
-		ServiceType:     *serviceType,
-		IsGKEInternalLB: *isGKEInternalLB,
-		Plus:            *plusEnabled,
+		ReleaseName:       cfg.releaseName,
+		Namespace:         ngfNamespace,
+		ChartPath:         cfg.chartPath,
+		ServiceType:       *serviceType,
+		IsGKEInternalLB:   *isGKEInternalLB,
+		Plus:              *plusEnabled,
+		PlusUsageEndpoint: *plusUsageEndpoint,
 	}
 
 	// if we aren't installing from the public charts, then set the custom images