Update nfr tests for cp/dp split

Author: bjee19

charts/nginx-gateway-fabric/values.yaml

@@ -372,19 +372,19 @@ nginx:
     # -- The termination grace period of the NGINX data plane pod.
     # terminationGracePeriodSeconds: 30
 
-    # -- Tolerations for the NGINX Gateway Fabric control plane pod.
+    # -- Tolerations for the NGINX data plane pod.
     # tolerations: []
 
-    # -- The nodeSelector of the NGINX Gateway Fabric control plane pod.
+    # -- The nodeSelector of the NGINX data plane pod.
     # nodeSelector: {}
 
-    # -- The affinity of the NGINX Gateway Fabric control plane pod.
+    # -- The affinity of the NGINX data plane pod.
     # affinity: {}
 
-    # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod.
+    # -- The topology spread constraints for the NGINX data plane pod.
     # topologySpreadConstraints: []
 
-    # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with
+    # -- extraVolumes for the NGINX data plane pod. Use in conjunction with
     # nginx.container.extraVolumeMounts mount additional volumes to the container.
     # extraVolumes: []
 

tests/Makefile

@@ -44,7 +44,7 @@ build-test-runner-image: ## Build conformance test runner image
 
 .PHONY: build-crossplane-image
 build-crossplane-image: ## Build the crossplane image
-	docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile ..
+	docker build --platform $(GOOS)/$(GOARCH) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile ..
 
 .PHONY: run-conformance-tests
 run-conformance-tests: ## Run conformance tests
@@ -105,7 +105,9 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
 	./scripts/sync-files-to-vm.sh
 
 .PHONY: nfr-test
-nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM
+nfr-test: GOARCH=amd64
+nfr-test: check-for-plus-usage-endpoint build-crossplane-image  ## Run the NFR tests on a GCP VM
+	./scripts/push-crossplane-image.sh
 	CI=$(CI) ./scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test

tests/framework/crossplane.go

@@ -38,6 +38,8 @@ type ExpectedNginxField struct {
 	ValueSubstringAllowed bool
 }
 
+const crossplaneImageName = "nginx-crossplane:latest"
+
 // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
 // and returns whether or not that field exists where it should.
 func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error {
@@ -144,11 +146,17 @@ func injectCrossplaneContainer(
 	k8sClient kubernetes.Interface,
 	timeout time.Duration,
 	ngfPodName,
-	namespace string,
+	namespace,
+	crossplaneImageRepo string,
 ) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
+	image := crossplaneImageName
+	if crossplaneImageRepo != "" {
+		image = crossplaneImageRepo + "/" + image
+	}
+
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ngfPodName,
@@ -160,8 +168,8 @@ func injectCrossplaneContainer(
 					TargetContainerName: "nginx",
 					EphemeralContainerCommon: core.EphemeralContainerCommon{
 						Name:            "crossplane",
-						Image:           "nginx-crossplane:latest",
-						ImagePullPolicy: "Never",
+						Image:           image,
+						ImagePullPolicy: "IfNotPresent",
 						Stdin:           true,
 						VolumeMounts: []core.VolumeMount{
 							{

tests/framework/info.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"runtime/debug"
 
+	. "github.com/onsi/ginkgo/v2"
 	core "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -81,3 +82,27 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
 
 	return
 }
+
+// AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed.
+func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) {
+	if CurrentSpecReport().Failed() {
+		var returnLogs string
+
+		nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout)
+
+		for _, nginxPodName := range nginxPodNames {
+			returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName)
+			nginxLogs, _ := rm.GetPodLogs(
+				namespace,
+				nginxPodName,
+				&core.PodLogOptions{Container: "nginx"},
+			)
+
+			returnLogs += fmt.Sprintf("  %s\n", nginxLogs)
+		}
+		AddReportEntry("Nginx Logs", returnLogs, ReportEntryVisibilityNever)
+
+		events := GetEvents(rm, namespace)
+		AddReportEntry("Test Events", events, ReportEntryVisibilityNever)
+	}
+}

tests/framework/prometheus.go

@@ -302,119 +302,6 @@ type Bucket struct {
 	Val int
 }
 
-// GetReloadCount gets the total number of nginx reloads.
-func GetReloadCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadCountWithStartTime gets the total number of nginx reloads from a start time to the current time.
-func GetReloadCountWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadErrsCountWithStartTime gets the total number of nginx reload errors from a start time to the current time.
-func GetReloadErrsCountWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadAvgTime gets the average time in milliseconds for nginx to reload.
-func GetReloadAvgTime(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+
-				` / `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadAvgTimeWithStartTime gets the average time in milliseconds for nginx to reload using a start time
-// to the current time to calculate.
-func GetReloadAvgTimeWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`(nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"} @ %[2]d)`+
-				` / `+
-				`(nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %[2]d)`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadBuckets gets the Buckets in millisecond intervals for nginx reloads.
-func GetReloadBuckets(promInstance PrometheusInstance, ngfPodName string) ([]Bucket, error) {
-	return getBuckets(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadBucketsWithStartTime gets the Buckets in millisecond intervals for nginx reloads from a start time
-// to the current time.
-func GetReloadBucketsWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) ([]Bucket, error) {
-	return getBuckets(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
 // GetEventsCount gets the NGF event batch processing count.
 func GetEventsCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
 	return getFirstValueOfVector(

tests/framework/resourcemanager.go

@@ -46,6 +46,8 @@ import (
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	v1 "sigs.k8s.io/gateway-api/apis/v1"
+
+	ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
 )
 
 // ResourceManager handles creating/updating/deleting Kubernetes resources.
@@ -647,6 +649,44 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app
 	return &deployment, nil
 }
 
+func (rm *ResourceManager) getGatewayClassNginxProxy(
+	namespace,
+	releaseName string,
+) (*ngfAPIv1alpha2.NginxProxy, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
+	defer cancel()
+
+	var proxy ngfAPIv1alpha2.NginxProxy
+	proxyName := releaseName + "-proxy-config"
+
+	if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil {
+		return nil, err
+	}
+
+	return &proxy, nil
+}
+
+// ScaleNginxDeployment scales the Nginx Deployment to the specified number of replicas.
+func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, replicas int32) error {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout)
+	defer cancel()
+
+	// If there is another NginxProxy which "overrides" the gateway class  one, then this won't work and
+	// may need refactoring.
+	proxy, err := rm.getGatewayClassNginxProxy(namespace, releaseName)
+	if err != nil {
+		return fmt.Errorf("error getting NginxProxy: %w", err)
+	}
+
+	proxy.Spec.Kubernetes.Deployment.Replicas = &replicas
+
+	if err = rm.K8sClient.Update(ctx, proxy); err != nil {
+		return fmt.Errorf("error updating NginxProxy: %w", err)
+	}
+
+	return nil
+}
+
 // GetEvents returns all Events in the specified namespace.
 func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
@@ -825,12 +865,14 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration(
 }
 
 // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON.
-func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace string) (*Payload, error) {
+// If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty.
+func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) {
 	if err := injectCrossplaneContainer(
 		rm.ClientGoClient,
 		rm.TimeoutConfig.UpdateTimeout,
 		nginxPodName,
 		namespace,
+		crossplaneImageRepo,
 	); err != nil {
 		return nil, err
 	}

tests/framework/timeout.go

@@ -43,7 +43,7 @@ func DefaultTimeoutConfig() TimeoutConfig {
 		CreateTimeout:           60 * time.Second,
 		UpdateTimeout:           60 * time.Second,
 		DeleteTimeout:           10 * time.Second,
-		DeleteNamespaceTimeout:  60 * time.Second,
+		DeleteNamespaceTimeout:  90 * time.Second,
 		GetTimeout:              10 * time.Second,
 		ManifestFetchTimeout:    10 * time.Second,
 		RequestTimeout:          10 * time.Second,

tests/scripts/push-crossplane-image.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+source scripts/vars.env
+
+docker tag nginx-crossplane:latest gcr.io/$GKE_PROJECT/nginx-crossplane:latest
+docker push gcr.io/$GKE_PROJECT/nginx-crossplane:latest

tests/suite/advanced_routing_test.go

@@ -48,6 +48,7 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun
 	})
 
 	AfterAll(func() {
+		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace)
 		cleanUpPortForward()
 
 		Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed())

tests/suite/client_settings_test.go

@@ -57,6 +57,7 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy"
 	})
 
 	AfterAll(func() {
+		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace)
 		cleanUpPortForward()
 
 		Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed())
@@ -109,7 +110,7 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy"
 
 			BeforeAll(func() {
 				var err error
-				conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace)
+				conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "")
 				Expect(err).ToNot(HaveOccurred())
 			})
 

tests/suite/dataplane_perf_test.go

@@ -88,6 +88,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"),
 	})
 
 	AfterAll(func() {
+		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace)
 		cleanUpPortForward()
 
 		Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed())

tests/suite/graceful_recovery_test.go

@@ -425,6 +425,7 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("graceful-recovery"),
 	})
 
 	AfterAll(func() {
+		framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name)
 		cleanUpPortForward()
 		Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed())

tests/suite/longevity_test.go

@@ -82,6 +82,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu
 		Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed())
 		Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed())
 
+		framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name)
 		Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed())
 	})