From 05672e5bdd81fa777b2b5c4c8e5e0bb0b7d2a056 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Wed, 9 Apr 2025 15:24:50 -0700 Subject: [PATCH 1/5] Update nfr tests for cp/dp split --- charts/nginx-gateway-fabric/values.yaml | 10 +- tests/Makefile | 6 +- tests/framework/crossplane.go | 14 +- tests/framework/info.go | 25 + tests/framework/prometheus.go | 113 ---- tests/framework/resourcemanager.go | 44 +- tests/framework/timeout.go | 2 +- tests/scripts/push-crossplane-image.sh | 8 + tests/suite/advanced_routing_test.go | 1 + tests/suite/client_settings_test.go | 3 +- tests/suite/dataplane_perf_test.go | 1 + tests/suite/graceful_recovery_test.go | 1 + tests/suite/longevity_test.go | 1 + .../suite/manifests/reconfig/cafe-routes.yaml | 6 +- .../scale/zero-downtime/values-affinity.yaml | 40 +- .../manifests/scale/zero-downtime/values.yaml | 18 +- tests/suite/reconfig_test.go | 509 +++++++----------- tests/suite/sample_test.go | 1 + tests/suite/scale_test.go | 178 +++--- tests/suite/scripts/longevity-wrk.sh | 11 +- tests/suite/snippets_filter_test.go | 3 +- tests/suite/system_suite_test.go | 2 +- tests/suite/tracing_test.go | 1 + tests/suite/upgrade_test.go | 11 +- tests/suite/upstream_settings_test.go | 5 +- 25 files changed, 459 insertions(+), 555 deletions(-) create mode 100755 tests/scripts/push-crossplane-image.sh diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 4ad0230669..cfb40a1d9f 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -372,19 +372,19 @@ nginx: # -- The termination grace period of the NGINX data plane pod. # terminationGracePeriodSeconds: 30 - # -- Tolerations for the NGINX Gateway Fabric control plane pod. + # -- Tolerations for the NGINX data plane pod. # tolerations: [] - # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. + # -- The nodeSelector of the NGINX data plane pod. # nodeSelector: {} - # -- The affinity of the NGINX Gateway Fabric control plane pod. + # -- The affinity of the NGINX data plane pod. # affinity: {} - # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. + # -- The topology spread constraints for the NGINX data plane pod. # topologySpreadConstraints: [] - # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with + # -- extraVolumes for the NGINX data plane pod. Use in conjunction with # nginx.container.extraVolumeMounts mount additional volumes to the container. # extraVolumes: [] diff --git a/tests/Makefile b/tests/Makefile index dd82633d76..eec81e0ad7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -44,7 +44,7 @@ build-test-runner-image: ## Build conformance test runner image .PHONY: build-crossplane-image build-crossplane-image: ## Build the crossplane image - docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile .. + docker build --platform $(GOOS)/$(GOARCH) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile .. .PHONY: run-conformance-tests run-conformance-tests: ## Run conformance tests @@ -105,7 +105,9 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM ./scripts/sync-files-to-vm.sh .PHONY: nfr-test -nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM +nfr-test: GOARCH=amd64 +nfr-test: check-for-plus-usage-endpoint build-crossplane-image ## Run the NFR tests on a GCP VM + ./scripts/push-crossplane-image.sh CI=$(CI) ./scripts/run-tests-gcp-vm.sh .PHONY: start-longevity-test diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go index f2ada703c5..02a16b6cb5 100644 --- a/tests/framework/crossplane.go +++ b/tests/framework/crossplane.go @@ -38,6 +38,8 @@ type ExpectedNginxField struct { ValueSubstringAllowed bool } +const crossplaneImageName = "nginx-crossplane:latest" + // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, // and returns whether or not that field exists where it should. func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error { @@ -144,11 +146,17 @@ func injectCrossplaneContainer( k8sClient kubernetes.Interface, timeout time.Duration, ngfPodName, - namespace string, + namespace, + crossplaneImageRepo string, ) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + image := crossplaneImageName + if crossplaneImageRepo != "" { + image = crossplaneImageRepo + "/" + image + } + pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: ngfPodName, @@ -160,8 +168,8 @@ func injectCrossplaneContainer( TargetContainerName: "nginx", EphemeralContainerCommon: core.EphemeralContainerCommon{ Name: "crossplane", - Image: "nginx-crossplane:latest", - ImagePullPolicy: "Never", + Image: image, + ImagePullPolicy: "IfNotPresent", Stdin: true, VolumeMounts: []core.VolumeMount{ { diff --git a/tests/framework/info.go b/tests/framework/info.go index 588b728631..c485edc9aa 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -4,6 +4,7 @@ import ( "fmt" "runtime/debug" + . "github.com/onsi/ginkgo/v2" core "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -81,3 +82,27 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { return } + +// AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed. +func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) { + if CurrentSpecReport().Failed() { + var returnLogs string + + nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout) + + for _, nginxPodName := range nginxPodNames { + returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName) + nginxLogs, _ := rm.GetPodLogs( + namespace, + nginxPodName, + &core.PodLogOptions{Container: "nginx"}, + ) + + returnLogs += fmt.Sprintf(" %s\n", nginxLogs) + } + AddReportEntry("Nginx Logs", returnLogs, ReportEntryVisibilityNever) + + events := GetEvents(rm, namespace) + AddReportEntry("Test Events", events, ReportEntryVisibilityNever) + } +} diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index fd7bf44624..3c3094712b 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -302,119 +302,6 @@ type Bucket struct { Val int } -// GetReloadCount gets the total number of nginx reloads. -func GetReloadCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) { - return getFirstValueOfVector( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`, - ngfPodName, - ), - promInstance, - ) -} - -// GetReloadCountWithStartTime gets the total number of nginx reloads from a start time to the current time. -func GetReloadCountWithStartTime( - promInstance PrometheusInstance, - ngfPodName string, - startTime time.Time, -) (float64, error) { - return getFirstValueOfVector( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+ - ` - `+ - `nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %d`, - ngfPodName, - startTime.Unix(), - ), - promInstance, - ) -} - -// GetReloadErrsCountWithStartTime gets the total number of nginx reload errors from a start time to the current time. -func GetReloadErrsCountWithStartTime( - promInstance PrometheusInstance, - ngfPodName string, - startTime time.Time, -) (float64, error) { - return getFirstValueOfVector( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"}`+ - ` - `+ - `nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"} @ %d`, - ngfPodName, - startTime.Unix(), - ), - promInstance, - ) -} - -// GetReloadAvgTime gets the average time in milliseconds for nginx to reload. -func GetReloadAvgTime(promInstance PrometheusInstance, ngfPodName string) (float64, error) { - return getFirstValueOfVector( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+ - ` / `+ - `nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`, - ngfPodName, - ), - promInstance, - ) -} - -// GetReloadAvgTimeWithStartTime gets the average time in milliseconds for nginx to reload using a start time -// to the current time to calculate. -func GetReloadAvgTimeWithStartTime( - promInstance PrometheusInstance, - ngfPodName string, - startTime time.Time, -) (float64, error) { - return getFirstValueOfVector( - fmt.Sprintf( - `(nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+ - ` - `+ - `nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"} @ %[2]d)`+ - ` / `+ - `(nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+ - ` - `+ - `nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %[2]d)`, - ngfPodName, - startTime.Unix(), - ), - promInstance, - ) -} - -// GetReloadBuckets gets the Buckets in millisecond intervals for nginx reloads. -func GetReloadBuckets(promInstance PrometheusInstance, ngfPodName string) ([]Bucket, error) { - return getBuckets( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`, - ngfPodName, - ), - promInstance, - ) -} - -// GetReloadBucketsWithStartTime gets the Buckets in millisecond intervals for nginx reloads from a start time -// to the current time. -func GetReloadBucketsWithStartTime( - promInstance PrometheusInstance, - ngfPodName string, - startTime time.Time, -) ([]Bucket, error) { - return getBuckets( - fmt.Sprintf( - `nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`+ - ` - `+ - `nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"} @ %d`, - ngfPodName, - startTime.Unix(), - ), - promInstance, - ) -} - // GetEventsCount gets the NGF event batch processing count. func GetEventsCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) { return getFirstValueOfVector( diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index 912c321090..f398e97375 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -46,6 +46,8 @@ import ( "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" + + ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" ) // ResourceManager handles creating/updating/deleting Kubernetes resources. @@ -647,6 +649,44 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app return &deployment, nil } +func (rm *ResourceManager) getGatewayClassNginxProxy( + namespace, + releaseName string, +) (*ngfAPIv1alpha2.NginxProxy, error) { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) + defer cancel() + + var proxy ngfAPIv1alpha2.NginxProxy + proxyName := releaseName + "-proxy-config" + + if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { + return nil, err + } + + return &proxy, nil +} + +// ScaleNginxDeployment scales the Nginx Deployment to the specified number of replicas. +func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, replicas int32) error { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout) + defer cancel() + + // If there is another NginxProxy which "overrides" the gateway class one, then this won't work and + // may need refactoring. + proxy, err := rm.getGatewayClassNginxProxy(namespace, releaseName) + if err != nil { + return fmt.Errorf("error getting NginxProxy: %w", err) + } + + proxy.Spec.Kubernetes.Deployment.Replicas = &replicas + + if err = rm.K8sClient.Update(ctx, proxy); err != nil { + return fmt.Errorf("error updating NginxProxy: %w", err) + } + + return nil +} + // GetEvents returns all Events in the specified namespace. func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) @@ -843,12 +883,14 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( } // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. -func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace string) (*Payload, error) { +// If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty. +func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) { if err := injectCrossplaneContainer( rm.ClientGoClient, rm.TimeoutConfig.UpdateTimeout, nginxPodName, namespace, + crossplaneImageRepo, ); err != nil { return nil, err } diff --git a/tests/framework/timeout.go b/tests/framework/timeout.go index 956b1699f3..8d8557622f 100644 --- a/tests/framework/timeout.go +++ b/tests/framework/timeout.go @@ -43,7 +43,7 @@ func DefaultTimeoutConfig() TimeoutConfig { CreateTimeout: 60 * time.Second, UpdateTimeout: 60 * time.Second, DeleteTimeout: 10 * time.Second, - DeleteNamespaceTimeout: 60 * time.Second, + DeleteNamespaceTimeout: 90 * time.Second, GetTimeout: 10 * time.Second, ManifestFetchTimeout: 10 * time.Second, RequestTimeout: 10 * time.Second, diff --git a/tests/scripts/push-crossplane-image.sh b/tests/scripts/push-crossplane-image.sh new file mode 100755 index 0000000000..3fe02d83bd --- /dev/null +++ b/tests/scripts/push-crossplane-image.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -eo pipefail + +source scripts/vars.env + +docker tag nginx-crossplane:latest gcr.io/$GKE_PROJECT/nginx-crossplane:latest +docker push gcr.io/$GKE_PROJECT/nginx-crossplane:latest diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 1359beb795..cffc5bad5b 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -48,6 +48,7 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 7bd5f4ed57..7a77c0dea9 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -57,6 +57,7 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) @@ -109,7 +110,7 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" BeforeAll(func() { var err error - conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace) + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") Expect(err).ToNot(HaveOccurred()) }) diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index aa34131db1..adedebf05f 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -88,6 +88,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index bd71551933..c2987610f7 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -425,6 +425,7 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("graceful-recovery"), }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index c768c71cb7..182c0fd676 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -82,6 +82,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed()) + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) }) diff --git a/tests/suite/manifests/reconfig/cafe-routes.yaml b/tests/suite/manifests/reconfig/cafe-routes.yaml index 006a8eba92..454d093892 100644 --- a/tests/suite/manifests/reconfig/cafe-routes.yaml +++ b/tests/suite/manifests/reconfig/cafe-routes.yaml @@ -5,7 +5,7 @@ metadata: spec: parentRefs: - name: gateway - namespace: default + namespace: reconfig sectionName: http hostnames: - "cafe.example.com" @@ -23,7 +23,7 @@ metadata: spec: parentRefs: - name: gateway - namespace: default + namespace: reconfig sectionName: https hostnames: - "cafe.example.com" @@ -43,8 +43,8 @@ metadata: spec: parentRefs: - name: gateway + namespace: reconfig sectionName: https - namespace: default hostnames: - "cafe.example.com" rules: diff --git a/tests/suite/manifests/scale/zero-downtime/values-affinity.yaml b/tests/suite/manifests/scale/zero-downtime/values-affinity.yaml index d9a0381b8e..ea19a27470 100644 --- a/tests/suite/manifests/scale/zero-downtime/values-affinity.yaml +++ b/tests/suite/manifests/scale/zero-downtime/values-affinity.yaml @@ -3,24 +3,26 @@ nginxGateway: preStop: exec: command: - - /usr/bin/gateway - - sleep - - --duration=40s + - /usr/bin/gateway + - sleep + - --duration=40s + terminationGracePeriodSeconds: 50 -nginx: - lifecycle: - preStop: - exec: - command: - - /bin/sleep - - "40" - -terminationGracePeriodSeconds: 50 -affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app.kubernetes.io/name: nginx-gateway +nginx: + pod: + terminationGracePeriodSeconds: 50 + container: + lifecycle: + preStop: + exec: + command: + - /bin/sleep + - "40" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: gateway-nginx diff --git a/tests/suite/manifests/scale/zero-downtime/values.yaml b/tests/suite/manifests/scale/zero-downtime/values.yaml index b4de7a5528..06f18b79a7 100644 --- a/tests/suite/manifests/scale/zero-downtime/values.yaml +++ b/tests/suite/manifests/scale/zero-downtime/values.yaml @@ -6,13 +6,15 @@ nginxGateway: - /usr/bin/gateway - sleep - --duration=40s + terminationGracePeriodSeconds: 50 nginx: - lifecycle: - preStop: - exec: - command: - - /bin/sleep - - "40" - -terminationGracePeriodSeconds: 50 + pod: + terminationGracePeriodSeconds: 50 + container: + lifecycle: + preStop: + exec: + command: + - /bin/sleep + - "40" diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index fb4d6c02ce..691497a01d 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -107,7 +107,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r return nil } - createResourcesGWLast := func(resourceCount int) { + createResources := func(resourceCount int) { ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout*5) defer cancel() @@ -140,44 +140,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r } Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name)).To(Succeed()) } - - Expect(resourceManager.ApplyFromFiles([]string{"reconfig/gateway.yaml"}, reconfigNamespace.Name)).To(Succeed()) - } - - createResourcesRoutesLast := func(resourceCount int) { - ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout*5) - defer cancel() - - for i := 1; i <= resourceCount; i++ { - ns := core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "namespace" + strconv.Itoa(i), - }, - } - Expect(k8sClient.Create(ctx, &ns)).To(Succeed()) - } - - Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe.yaml")).To(Succeed()) - - for i := 1; i <= resourceCount; i++ { - ns := core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "namespace" + strconv.Itoa(i), - }, - } - Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name)).To(Succeed()) - } - - Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) - Expect(resourceManager.ApplyFromFiles( - []string{ - "reconfig/cafe-secret.yaml", - "reconfig/reference-grant.yaml", - "reconfig/gateway.yaml", - }, - reconfigNamespace.Name)).To(Succeed()) - - Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe-routes.yaml")).To(Succeed()) } checkResourceCreation := func(resourceCount int) error { @@ -223,131 +185,64 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r return err } - getTimeStampFromLogLine := func(logLine string) string { - var timeStamp string - - timeStamp = strings.Split(logLine, "\"ts\":\"")[1] - // sometimes the log message will contain information on a "logger" followed by the "msg" - // while other times the "logger" will be omitted - timeStamp = strings.Split(timeStamp, "\",\"msg\"")[0] - timeStamp = strings.Split(timeStamp, "\",\"logger\"")[0] - - return timeStamp - } - - calculateTimeDifferenceBetweenLogLines := func(firstLine, secondLine string) (int, error) { - layout := time.RFC3339 - - firstTS := getTimeStampFromLogLine(firstLine) - secondTS := getTimeStampFromLogLine(secondLine) - - parsedTS1, err := time.Parse(layout, firstTS) - if err != nil { - return 0, err - } - - parsedTS2, err := time.Parse(layout, secondTS) - if err != nil { - return 0, err - } - - return int(parsedTS2.Sub(parsedTS1).Seconds()), nil - } + checkNginxConfIsPopulated := func(nginxPodName string, resourceCount int) error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout*2) + defer cancel() - calculateTimeToReadyAverage := func(ngfLogs string) (string, error) { - var reconcilingLine, nginxReloadLine string - const maxCount = 5 - - var times [maxCount]int - var count int - - // parse the logs until it reaches a reconciling log line for a gateway resource, then it compares that - // timestamp to the next NGINX configuration update. When it reaches the NGINX configuration update line, - // it will reset the reconciling log line and set it to the next reconciling log line. - for _, line := range strings.Split(ngfLogs, "\n") { - if reconcilingLine == "" && - strings.Contains(line, "Reconciling the resource\",\"controller\"") && - strings.Contains(line, "\"controllerGroup\":\"gateway.networking.k8s.io\"") { - reconcilingLine = line + index := 1 + conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + for index <= resourceCount { + namespace := "namespace" + strconv.Itoa(resourceCount) + expUpstream := framework.ExpectedNginxField{ + Directive: "upstream", + Value: namespace + "_coffee" + namespace + "_80", + File: "http.conf", } - if strings.Contains(line, "NGINX configuration was successfully updated") && reconcilingLine != "" { - nginxReloadLine = line - - timeDifference, err := calculateTimeDifferenceBetweenLogLines(reconcilingLine, nginxReloadLine) - if err != nil { - return "", err - } - reconcilingLine = "" - - times[count] = timeDifference - count++ - if count == maxCount-1 { - break + // each call to ValidateNginxFieldExists takes about 1ms + if err := framework.ValidateNginxFieldExists(conf, expUpstream); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("error validating nginx conf was generated in "+namespace+": %w", err.Error()) + default: + // each call to GetNginxConfig takes about 70ms + conf, _ = resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + continue } } - } - var sum float64 - for _, time := range times { - sum += float64(time) + index++ } - avgTime := sum / float64(count+1) - - if avgTime < 1 { - return "< 1", nil - } - - return strconv.FormatFloat(avgTime, 'f', -1, 64), nil + return nil } - calculateTimeToReadyTotal := func(ngfLogs, startingLogSubstring string) (string, error) { - var firstLine, lastLine string - for _, line := range strings.Split(ngfLogs, "\n") { - if firstLine == "" && strings.Contains(line, startingLogSubstring) { - firstLine = line - } + calculateTimeToReadyTotal := func(nginxPodName string, startTime time.Time, resourceCount int) string { + Expect(checkNginxConfIsPopulated(nginxPodName, resourceCount)).To(Succeed()) + stopTime := time.Now() - if strings.Contains(line, "NGINX configuration was successfully updated") { - lastLine = line - } - } - - timeToReadyTotal, err := calculateTimeDifferenceBetweenLogLines(firstLine, lastLine) - if err != nil { - return "", err - } + stringTimeToReadyTotal := strconv.Itoa(int(stopTime.Sub(startTime).Seconds())) - stringTimeToReadyTotal := strconv.Itoa(timeToReadyTotal) if stringTimeToReadyTotal == "0" { stringTimeToReadyTotal = "< 1" } - return stringTimeToReadyTotal, nil + return stringTimeToReadyTotal } - deployNGFReturnsNGFPodNameAndStartTime := func() (string, time.Time) { - var startTime time.Time - + collectMetrics := func( + resourceCount int, + ngfPodName string, + startTime time.Time, + ) reconfigTestResults { getStartTime := func() time.Time { return startTime } modifyStartTime := func() { startTime = startTime.Add(500 * time.Millisecond) } - cfg := getDefaultSetupCfg() - cfg.nfr = true - setup(cfg) - - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) - Expect(err).ToNot(HaveOccurred()) - Expect(podNames).To(HaveLen(1)) - ngfPodName := podNames[0] - startTime = time.Now() - queries := []string{ fmt.Sprintf(`container_memory_usage_bytes{pod="%s",container="nginx-gateway"}`, ngfPodName), fmt.Sprintf(`container_cpu_usage_seconds_total{pod="%s",container="nginx-gateway"}`, ngfPodName), // We don't need to check all nginx_gateway_fabric_* metrics, as they are collected at the same time - fmt.Sprintf(`nginx_gateway_fabric_nginx_reloads_total{pod="%s"}`, ngfPodName), + fmt.Sprintf(`nginx_gateway_fabric_event_batch_processing_milliseconds_sum{pod="%s"}`, ngfPodName), } for _, q := range queries { @@ -361,16 +256,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } - return ngfPodName, startTime - } - - collectMetrics := func( - testDescription string, - resourceCount int, - timeToReadyStartingLogSubstring string, - ngfPodName string, - startTime time.Time, - ) { time.Sleep(2 * scrapeInterval) endTime := time.Now() @@ -388,12 +273,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r getEndTime := func() time.Time { return endTime } noOpModifier := func() {} - queries := []string{ - fmt.Sprintf(`container_memory_usage_bytes{pod="%s",container="nginx-gateway"}`, ngfPodName), - // We don't need to check all nginx_gateway_fabric_* metrics, as they are collected at the same time - fmt.Sprintf(`nginx_gateway_fabric_nginx_reloads_total{pod="%s"}`, ngfPodName), - } - for _, q := range queries { Eventually( framework.CreateMetricExistChecker( @@ -407,21 +286,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r checkNGFContainerLogsForErrors(ngfPodName) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, reconfigNamespace.Name, timeoutConfig.GetTimeout) - Expect(err).ToNot(HaveOccurred()) - Expect(nginxPodNames).To(HaveLen(1)) - - nginxErrorLogs := getNginxErrorLogs(nginxPodNames[0], reconfigNamespace.Name) - - reloadCount, err := framework.GetReloadCount(promInstance, ngfPodName) - Expect(err).ToNot(HaveOccurred()) - - reloadAvgTime, err := framework.GetReloadAvgTime(promInstance, ngfPodName) - Expect(err).ToNot(HaveOccurred()) - - reloadBuckets, err := framework.GetReloadBuckets(promInstance, ngfPodName) - Expect(err).ToNot(HaveOccurred()) - eventsCount, err := framework.GetEventsCount(promInstance, ngfPodName) Expect(err).ToNot(HaveOccurred()) @@ -431,158 +295,156 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r eventsBuckets, err := framework.GetEventsBuckets(promInstance, ngfPodName) Expect(err).ToNot(HaveOccurred()) - logs, err := resourceManager.GetPodLogs(ngfNamespace, ngfPodName, &core.PodLogOptions{ - Container: "nginx-gateway", - }) - Expect(err).ToNot(HaveOccurred()) - - // FIXME (bjee19): https://github.com/nginx/nginx-gateway-fabric/issues/2374 - // Find a way to calculate time to ready metrics without having to rely on specific log lines. - timeToReadyTotal, err := calculateTimeToReadyTotal(logs, timeToReadyStartingLogSubstring) - Expect(err).ToNot(HaveOccurred()) - - timeToReadyAvgSingle, err := calculateTimeToReadyAverage(logs) - Expect(err).ToNot(HaveOccurred()) - results := reconfigTestResults{ - TestDescription: testDescription, - EventsBuckets: eventsBuckets, - ReloadBuckets: reloadBuckets, - NumResources: resourceCount, - TimeToReadyTotal: timeToReadyTotal, - TimeToReadyAvgSingle: timeToReadyAvgSingle, - NGINXReloads: int(reloadCount), - NGINXReloadAvgTime: int(reloadAvgTime), - NGINXErrorLogs: nginxErrorLogs, - EventsCount: int(eventsCount), - EventsAvgTime: int(eventsAvgTime), + EventsBuckets: eventsBuckets, + NumResources: resourceCount, + EventsCount: int(eventsCount), + EventsAvgTime: int(eventsAvgTime), } - err = writeReconfigResults(outFile, results) - Expect(err).ToNot(HaveOccurred()) + return results } When("resources exist before startup", func() { testDescription := "Test 1: Resources exist before startup" - - It("gathers metrics after creating 30 resources", func() { - resourceCount := 30 - timeToReadyStartingLogSubstring := "Starting NGINX Gateway Fabric" - - createResourcesGWLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) - - It("gathers metrics after creating 150 resources", func() { - resourceCount := 150 - timeToReadyStartingLogSubstring := "Starting NGINX Gateway Fabric" - - createResourcesGWLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) + timeToReadyDescription := "From when NGF starts to when the NGINX configuration is fully configured" + DescribeTable(testDescription, + func(resourceCount int) { + createResources(resourceCount) + Expect(resourceManager.ApplyFromFiles([]string{"reconfig/gateway.yaml"}, reconfigNamespace.Name)).To(Succeed()) + Expect(checkResourceCreation(resourceCount)).To(Succeed()) + + cfg := getDefaultSetupCfg() + cfg.nfr = true + setup(cfg) + + podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(podNames).To(HaveLen(1)) + ngfPodName := podNames[0] + startTime := time.Now() + + var nginxPodNames []string + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + reconfigNamespace.Name, + timeoutConfig.GetStatusTimeout, + ) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + WithPolling(500 * time.Millisecond). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + timeToReadyTotal := calculateTimeToReadyTotal(nginxPodName, startTime, resourceCount) + + nginxErrorLogs := getNginxErrorLogs(nginxPodNames[0], reconfigNamespace.Name) + + results := collectMetrics( + resourceCount, + ngfPodName, + startTime, + ) + + results.NGINXErrorLogs = nginxErrorLogs + results.TimeToReadyTotal = timeToReadyTotal + results.TestDescription = testDescription + results.TimeToReadyDescription = timeToReadyDescription + + err = writeReconfigResults(outFile, results) + Expect(err).ToNot(HaveOccurred()) + }, + Entry("gathers metrics after creating 30 resources", 30), + Entry("gathers metrics after creating 150 resources", 150), + ) }) When("NGF and Gateway resource are deployed first", func() { - testDescription := "Test 2: Start NGF, deploy Gateway, create many resources attached to GW" - - It("gathers metrics after creating 30 resources", func() { - resourceCount := 30 - timeToReadyStartingLogSubstring := "Reconciling the resource\",\"controller\":\"httproute\"" - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - createResourcesRoutesLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) - - It("gathers metrics after creating 150 resources", func() { - resourceCount := 150 - timeToReadyStartingLogSubstring := "Reconciling the resource\",\"controller\":\"httproute\"" - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - createResourcesRoutesLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) - }) - - When("NGF and resources are deployed first", func() { - testDescription := "Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway" - - It("gathers metrics after creating 30 resources", func() { - resourceCount := 30 - timeToReadyStartingLogSubstring := "Reconciling the resource\",\"controller\":\"gateway\"" - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - createResourcesGWLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) - - It("gathers metrics after creating 150 resources", func() { - resourceCount := 150 - timeToReadyStartingLogSubstring := "Reconciling the resource\",\"controller\":\"gateway\"" - - ngfPodName, startTime := deployNGFReturnsNGFPodNameAndStartTime() - - createResourcesGWLast(resourceCount) - Expect(checkResourceCreation(resourceCount)).To(Succeed()) - - collectMetrics( - testDescription, - resourceCount, - timeToReadyStartingLogSubstring, - ngfPodName, - startTime, - ) - }) + testDescription := "Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, " + + "create many resources attached to GW" + timeToReadyDescription := "From when NGINX receives the first configuration created by NGF to " + + "when the NGINX configuration is fully configured" + DescribeTable(testDescription, + func(resourceCount int) { + cfg := getDefaultSetupCfg() + cfg.nfr = true + setup(cfg) + + podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(podNames).To(HaveLen(1)) + ngfPodName := podNames[0] + + Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) + Expect(resourceManager.ApplyFromFiles([]string{"reconfig/gateway.yaml"}, reconfigNamespace.Name)).To(Succeed()) + + var nginxPodNames []string + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + reconfigNamespace.Name, + timeoutConfig.GetStatusTimeout, + ) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + // this checks if NGF has established a connection with agent and sent over the first nginx conf + Eventually( + func() bool { + conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + // a default upstream NGF creates + defaultUpstream := framework.ExpectedNginxField{ + Directive: "upstream", + Value: "invalid-backend-ref", + File: "http.conf", + } + + return framework.ValidateNginxFieldExists(conf, defaultUpstream) == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + startTime := time.Now() + + createResources(resourceCount) + Expect(checkResourceCreation(resourceCount)).To(Succeed()) + + timeToReadyTotal := calculateTimeToReadyTotal(nginxPodName, startTime, resourceCount) + + nginxErrorLogs := getNginxErrorLogs(nginxPodName, reconfigNamespace.Name) + + results := collectMetrics( + resourceCount, + ngfPodName, + startTime, + ) + + results.NGINXErrorLogs = nginxErrorLogs + results.TimeToReadyTotal = timeToReadyTotal + results.TestDescription = testDescription + results.TimeToReadyDescription = timeToReadyDescription + + err = writeReconfigResults(outFile, results) + Expect(err).ToNot(HaveOccurred()) + }, + Entry("gathers metrics after creating 30 resources", 30), + Entry("gathers metrics after creating 150 resources", 150), + ) }) AfterEach(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, reconfigNamespace.Name) + Expect(cleanupResources()).Should(Succeed()) teardown(releaseName) }) @@ -600,32 +462,23 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) type reconfigTestResults struct { - TestDescription string - TimeToReadyTotal string - TimeToReadyAvgSingle string - NGINXErrorLogs string - EventsBuckets []framework.Bucket - ReloadBuckets []framework.Bucket - NumResources int - NGINXReloads int - NGINXReloadAvgTime int - EventsCount int - EventsAvgTime int + TestDescription string + TimeToReadyTotal string + TimeToReadyDescription string + NGINXErrorLogs string + EventsBuckets []framework.Bucket + NumResources int + EventsCount int + EventsAvgTime int } const reconfigResultTemplate = ` ## {{ .TestDescription }} - NumResources {{ .NumResources }} -### Reloads and Time to Ready +### Time to Ready +Time To Ready Description: {{ .TimeToReadyDescription }} - TimeToReadyTotal: {{ .TimeToReadyTotal }}s -- TimeToReadyAvgSingle: {{ .TimeToReadyAvgSingle }}s -- NGINX Reloads: {{ .NGINXReloads }} -- NGINX Reload Average Time: {{ .NGINXReloadAvgTime }}ms -- Reload distribution: -{{- range .ReloadBuckets }} - - {{ .Le }}ms: {{ .Val }} -{{- end }} ### Event Batch Processing diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index d4c265ec37..191d20134d 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -46,6 +46,7 @@ var _ = Describe("Basic test example", Label("functional"), func() { }) AfterEach(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 4dadb97d60..8a04a088e7 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -119,31 +119,17 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { type scaleTestResults struct { Name string EventsBuckets []framework.Bucket - ReloadBuckets []framework.Bucket EventsAvgTime int EventsCount int NGFContainerRestarts int NGFErrors int NginxContainerRestarts int NginxErrors int - ReloadAvgTime int - ReloadCount int - ReloadErrsCount int } const scaleResultTemplate = ` ## Test {{ .Name }} -### Reloads - -- Total: {{ .ReloadCount }} -- Total Errors: {{ .ReloadErrsCount }} -- Average Time: {{ .ReloadAvgTime }}ms -- Reload distribution: -{{- range .ReloadBuckets }} - - {{ .Le }}ms: {{ .Val }} -{{- end }} - ### Event Batch Processing - Total: {{ .EventsCount }} @@ -176,12 +162,14 @@ The logs are attached only if there are errors. } checkLogErrors := func( - containerName string, + containerName, + podName, + namespace, + fileName string, substrings []string, ignoredSubstrings []string, - fileName string, ) int { - logs, err := resourceManager.GetPodLogs(ngfNamespace, ngfPodName, &core.PodLogOptions{ + logs, err := resourceManager.GetPodLogs(namespace, podName, &core.PodLogOptions{ Container: containerName, }) Expect(err).ToNot(HaveOccurred()) @@ -237,7 +225,7 @@ The logs are attached only if there are errors. fmt.Sprintf(`container_memory_usage_bytes{pod="%s",container="nginx-gateway"}`, ngfPodName), fmt.Sprintf(`container_cpu_usage_seconds_total{pod="%s",container="nginx-gateway"}`, ngfPodName), // We don't need to check all nginx_gateway_fabric_* metrics, as they are collected at the same time - fmt.Sprintf(`nginx_gateway_fabric_nginx_reloads_total{pod="%s"}`, ngfPodName), + fmt.Sprintf(`nginx_gateway_fabric_event_batch_processing_milliseconds_sum{pod="%s"}`, ngfPodName), } for _, q := range queries { @@ -280,7 +268,7 @@ The logs are attached only if there are errors. queries = []string{ fmt.Sprintf(`container_memory_usage_bytes{pod="%s",container="nginx-gateway"}`, ngfPodName), // We don't need to check all nginx_gateway_fabric_* metrics, as they are collected at the same time - fmt.Sprintf(`nginx_gateway_fabric_nginx_reloads_total{pod="%s"}`, ngfPodName), + fmt.Sprintf(`nginx_gateway_fabric_event_batch_processing_milliseconds_sum{pod="%s"}`, ngfPodName), } for _, q := range queries { @@ -337,18 +325,6 @@ The logs are attached only if there are errors. Expect(os.Remove(cpuCSV)).To(Succeed()) - reloadCount, err := framework.GetReloadCountWithStartTime(promInstance, ngfPodName, startTime) - Expect(err).ToNot(HaveOccurred()) - - reloadErrsCount, err := framework.GetReloadErrsCountWithStartTime(promInstance, ngfPodName, startTime) - Expect(err).ToNot(HaveOccurred()) - - reloadAvgTime, err := framework.GetReloadAvgTimeWithStartTime(promInstance, ngfPodName, startTime) - Expect(err).ToNot(HaveOccurred()) - - reloadBuckets, err := framework.GetReloadBucketsWithStartTime(promInstance, ngfPodName, startTime) - Expect(err).ToNot(HaveOccurred()) - eventsCount, err := framework.GetEventsCountWithStartTime(promInstance, ngfPodName, startTime) Expect(err).ToNot(HaveOccurred()) @@ -362,43 +338,53 @@ The logs are attached only if there are errors. ngfErrors := checkLogErrors( "nginx-gateway", + ngfPodName, + ngfNamespace, + filepath.Join(testResultsDir, framework.CreateResultsFilename("log", "ngf", *plusEnabled)), []string{"error"}, []string{`"logger":"usageReporter`}, // ignore usageReporter errors - filepath.Join(testResultsDir, framework.CreateResultsFilename("log", "ngf", *plusEnabled)), ) + + nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(nginxPodNames).To(HaveLen(1)) + + nginxPodName := nginxPodNames[0] + nginxErrors := checkLogErrors( "nginx", + nginxPodName, + namespace, + filepath.Join(testResultsDir, framework.CreateResultsFilename("log", "nginx", *plusEnabled)), []string{framework.ErrorNGINXLog, framework.EmergNGINXLog, framework.CritNGINXLog, framework.AlertNGINXLog}, nil, - filepath.Join(testResultsDir, framework.CreateResultsFilename("log", "nginx", *plusEnabled)), ) // Check container restarts - pod, err := resourceManager.GetPod(ngfNamespace, ngfPodName) + ngfPod, err := resourceManager.GetPod(ngfNamespace, ngfPodName) + Expect(err).ToNot(HaveOccurred()) + + nginxPod, err := resourceManager.GetPod(namespace, nginxPodName) Expect(err).ToNot(HaveOccurred()) - findRestarts := func(name string) int { + findRestarts := func(containerName string, pod *core.Pod) int { for _, containerStatus := range pod.Status.ContainerStatuses { - if containerStatus.Name == name { + if containerStatus.Name == containerName { return int(containerStatus.RestartCount) } } - Fail(fmt.Sprintf("container %s not found", name)) + Fail(fmt.Sprintf("container %s not found", containerName)) return 0 } - ngfRestarts := findRestarts("nginx-gateway") - nginxRestarts := findRestarts("nginx") + ngfRestarts := findRestarts("nginx-gateway", ngfPod) + nginxRestarts := findRestarts("nginx", nginxPod) // Write results results := scaleTestResults{ Name: testName, - ReloadCount: int(reloadCount), - ReloadErrsCount: int(reloadErrsCount), - ReloadAvgTime: int(reloadAvgTime), - ReloadBuckets: reloadBuckets, EventsCount: int(eventsCount), EventsAvgTime: int(eventsAvgTime), EventsBuckets: eventsBuckets, @@ -428,6 +414,22 @@ The logs are attached only if there are errors. for i := range len(objects.ScaleIterationGroups) { Expect(resourceManager.Apply(objects.ScaleIterationGroups[i])).To(Succeed()) + if i == 0 { + var nginxPodNames []string + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + setUpPortForward(nginxPodName, namespace) + } + var url string if protocol == "http" && portFwdPort != 0 { url = fmt.Sprintf("%s://%d.example.com:%d", protocol, i, portFwdPort) @@ -441,7 +443,7 @@ The logs are attached only if there are errors. Eventually( framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), - ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) + ).WithTimeout(6 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) ttr := time.Since(startCheck) @@ -466,6 +468,21 @@ The logs are attached only if there are errors. Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + var nginxPodNames []string + var err error + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + setUpPortForward(nginxPodName, namespace) + var url string if portFwdPort != 0 { url = fmt.Sprintf("http://hello.example.com:%d", portFwdPort) @@ -598,6 +615,21 @@ The logs are attached only if there are errors. Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + var nginxPodNames []string + var err error + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + setUpPortForward(nginxPodName, namespace) + var port int if portFwdPort != 0 { port = portFwdPort @@ -611,7 +643,7 @@ The logs are attached only if there are errors. text := fmt.Sprintf("\n## Test %s\n\n", testName) - _, err := fmt.Fprint(outFile, text) + _, err = fmt.Fprint(outFile, text) Expect(err).ToNot(HaveOccurred()) run := func(t framework.Target) { @@ -650,8 +682,10 @@ The logs are attached only if there are errors. }) AfterEach(func() { - teardown(releaseName) + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) + teardown(releaseName) }) AfterAll(func() { @@ -678,13 +712,13 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim } var ( - outFile *os.File - resultsDir string - ngfDeploymentName string - ns core.Namespace - metricsCh chan *metricsResults + outFile *os.File + resultsDir string + ns core.Namespace + metricsCh chan *metricsResults - files = []string{ + numCoffeeAndTeaPods = 20 + files = []string{ "scale/zero-downtime/cafe.yaml", "scale/zero-downtime/cafe-secret.yaml", "scale/zero-downtime/gateway-1.yaml", @@ -825,12 +859,12 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim numReplicas int }{ { - name: "One NGF Pod runs per node", + name: "One NGINX Pod runs per node", valuesFile: "manifests/scale/zero-downtime/values-affinity.yaml", numReplicas: 12, // equals number of nodes }, { - name: "Multiple NGF Pods run per node", + name: "Multiple NGINX Pods run per node", valuesFile: "manifests/scale/zero-downtime/values.yaml", numReplicas: 24, // twice the number of nodes }, @@ -843,19 +877,33 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim cfg.nfr = true setup(cfg, "--values", test.valuesFile) - deploy, err := resourceManager.GetNGFDeployment(ngfNamespace, releaseName) - Expect(err).ToNot(HaveOccurred()) - ngfDeploymentName = deploy.GetName() - Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + var nginxPodNames []string + var err error + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + setUpPortForward(nginxPodName, ns.Name) + _, err = fmt.Fprintf(outFile, "\n## %s Test Results\n", test.name) Expect(err).ToNot(HaveOccurred()) }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + cleanUpPortForward() + teardown(releaseName) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) }) @@ -882,8 +930,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim // scale NGF up one at a time for i := 2; i <= test.numReplicas; i++ { - Eventually(resourceManager.ScaleDeployment). - WithArguments(ngfNamespace, ngfDeploymentName, int32(i)). + Eventually(resourceManager.ScaleNginxDeployment). + WithArguments(ngfNamespace, releaseName, int32(i)). WithTimeout(timeoutConfig.UpdateTimeout). WithPolling(500 * time.Millisecond). Should(Succeed()) @@ -893,7 +941,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout) - Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ngfNamespace, i)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ns.Name, i+numCoffeeAndTeaPods)).To(Succeed()) Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed()) cancel() @@ -935,8 +983,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim // scale NGF down one at a time currentGen := test.numReplicas for i := test.numReplicas - 1; i >= 1; i-- { - Eventually(resourceManager.ScaleDeployment). - WithArguments(ngfNamespace, ngfDeploymentName, int32(i)). + Eventually(resourceManager.ScaleNginxDeployment). + WithArguments(ngfNamespace, releaseName, int32(i)). WithTimeout(timeoutConfig.UpdateTimeout). WithPolling(500 * time.Millisecond). Should(Succeed()) @@ -1005,7 +1053,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim // allow traffic flow to start time.Sleep(2 * time.Second) - Expect(resourceManager.ScaleDeployment(ngfNamespace, ngfDeploymentName, int32(test.numReplicas))).To(Succeed()) + Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(test.numReplicas))).To(Succeed()) Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-2.yaml"}, ns.Name)).To(Succeed()) checkGatewayListeners(3) @@ -1037,7 +1085,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim // allow traffic flow to start time.Sleep(2 * time.Second) - Expect(resourceManager.ScaleDeployment(ngfNamespace, ngfDeploymentName, int32(1))).To(Succeed()) + Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(1))).To(Succeed()) Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-1.yaml"}, ns.Name)).To(Succeed()) checkGatewayListeners(2) diff --git a/tests/suite/scripts/longevity-wrk.sh b/tests/suite/scripts/longevity-wrk.sh index e7d3a6b23a..1165cfa6b5 100755 --- a/tests/suite/scripts/longevity-wrk.sh +++ b/tests/suite/scripts/longevity-wrk.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash -SVC_IP=$(kubectl -n nginx-gateway get svc ngf-longevity-nginx-gateway-fabric -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +while true; do + SVC_IP=$(kubectl -n longevity get svc gateway-nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [[ -n $SVC_IP ]]; then + echo "Service IP assigned: $SVC_IP" + break + fi + + echo "Still waiting for nginx Service IP..." + sleep 5 +done echo "${SVC_IP} cafe.example.com" | sudo tee -a /etc/hosts diff --git a/tests/suite/snippets_filter_test.go b/tests/suite/snippets_filter_test.go index 397f1e8c58..de6583d79b 100644 --- a/tests/suite/snippets_filter_test.go +++ b/tests/suite/snippets_filter_test.go @@ -67,6 +67,7 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) Expect(resourceManager.DeleteFromFiles(snippetsFilter, namespace)).To(Succeed()) }) @@ -119,7 +120,7 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter BeforeAll(func() { var err error - conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace) + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") Expect(err).ToNot(HaveOccurred()) }) diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 39e9e15524..a8e558da6b 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -348,7 +348,7 @@ var _ = AfterSuite(func() { AddReportEntry("Events", events, ReportEntryVisibilityNever) logs = framework.GetLogs(resourceManager, ngfNamespace, releaseName) - AddReportEntry("Logs", logs, ReportEntryVisibilityNever) + AddReportEntry("NGF Logs", logs, ReportEntryVisibilityNever) labelFilter := GinkgoLabelFilter() if !strings.Contains(labelFilter, "longevity-setup") { diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index 19b8a55f5f..f56a083426 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -110,6 +110,7 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac }) AfterEach(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) output, err := framework.UninstallCollector(resourceManager) Expect(err).ToNot(HaveOccurred(), string(output)) diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index d0dec2fc15..a62de97356 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -67,7 +67,12 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - var err error + nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(nginxPodNames).To(HaveLen(1)) + + setUpPortForward(nginxPodNames[0], ns.Name) + resultsDir, err = framework.CreateResultsDir("ngf-upgrade", version) Expect(err).ToNot(HaveOccurred()) @@ -78,12 +83,16 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }) AfterEach(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + cleanUpPortForward() + Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) resultsFile.Close() }) It("upgrades NGF with zero downtime", func() { + Skip("Skipping test until version 2.1.0 since 2.0.0 is a breaking change") nginxImage := *nginxImageRepository if *plusEnabled { nginxImage = *nginxPlusImageRepository diff --git a/tests/suite/upstream_settings_test.go b/tests/suite/upstream_settings_test.go index 4243a432c9..f2b02b1059 100644 --- a/tests/suite/upstream_settings_test.go +++ b/tests/suite/upstream_settings_test.go @@ -61,6 +61,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic }) AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) @@ -130,7 +131,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic BeforeAll(func() { var err error - conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace) + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") Expect(err).ToNot(HaveOccurred()) }) @@ -310,7 +311,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic BeforeAll(func() { var err error - conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace) + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") Expect(err).ToNot(HaveOccurred()) }) From 718d5a61c4dcbe7c01c768603d94b47e48d32de3 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Tue, 22 Apr 2025 13:39:29 -0700 Subject: [PATCH 2/5] Add login to GAR to workflow --- .github/workflows/nfr.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index e10ef2c0e7..4fea6b5bee 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -92,6 +92,13 @@ jobs: workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }} service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + - name: Login to GAR + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 with: From ff21c18fb4a8d17988ed1dc6009c5d66df108b19 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Tue, 22 Apr 2025 15:20:12 -0600 Subject: [PATCH 3/5] Fix handler tests --- internal/mode/static/handler_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index 6d62be42b3..ec5bfa437d 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -249,8 +249,6 @@ var _ = Describe("eventHandler", func() { handler.HandleEventBatch(context.Background(), logr.Discard(), batch) checkUpsertEventExpectations(e) - Expect(fakeProvisioner.RegisterGatewayCallCount()).Should(Equal(0)) - Expect(fakeGenerator.GenerateCallCount()).Should(Equal(0)) // status update should still occur for GatewayClasses Eventually( func() int { @@ -305,7 +303,7 @@ var _ = Describe("eventHandler", func() { Eventually( func() int { return fakeStatusUpdater.UpdateGroupCallCount() - }).Should(Equal(1)) + }).Should(BeNumerically(">", 1)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) Expect(name).To(Equal(groupControlPlane)) @@ -324,7 +322,7 @@ var _ = Describe("eventHandler", func() { Eventually( func() int { return fakeStatusUpdater.UpdateGroupCallCount() - }).Should(Equal(1)) + }).Should(BeNumerically(">", 1)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) Expect(name).To(Equal(groupControlPlane)) @@ -356,7 +354,7 @@ var _ = Describe("eventHandler", func() { Eventually( func() int { return fakeStatusUpdater.UpdateGroupCallCount() - }).Should(Equal(1)) + }).Should(BeNumerically(">", 1)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) Expect(name).To(Equal(groupControlPlane)) From 2829f3b672674132a1d409b6f1849adea2a5a885 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Tue, 22 Apr 2025 14:34:15 -0700 Subject: [PATCH 4/5] Change gcr.io to us-docker --- tests/scripts/push-crossplane-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/push-crossplane-image.sh b/tests/scripts/push-crossplane-image.sh index 3fe02d83bd..31bd06d2f6 100755 --- a/tests/scripts/push-crossplane-image.sh +++ b/tests/scripts/push-crossplane-image.sh @@ -4,5 +4,5 @@ set -eo pipefail source scripts/vars.env -docker tag nginx-crossplane:latest gcr.io/$GKE_PROJECT/nginx-crossplane:latest -docker push gcr.io/$GKE_PROJECT/nginx-crossplane:latest +docker tag nginx-crossplane:latest us-docker.pkg.dev/$GKE_PROJECT/nginx-gateway-fabric/nginx-crossplane:latest +docker push us-docker.pkg.dev/$GKE_PROJECT/nginx-gateway-fabric/nginx-crossplane:latest From 6736cc8df124d8b51b0b04cf7351e943c9085048 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Tue, 22 Apr 2025 15:23:52 -0700 Subject: [PATCH 5/5] Update nginx crossplane repo path --- tests/suite/reconfig_test.go | 6 +++--- tests/suite/system_suite_test.go | 31 +++++++++++++++++-------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index 691497a01d..a28596e5ad 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -190,7 +190,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r defer cancel() index := 1 - conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, nginxCrossplanePath) for index <= resourceCount { namespace := "namespace" + strconv.Itoa(resourceCount) expUpstream := framework.ExpectedNginxField{ @@ -206,7 +206,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r return fmt.Errorf("error validating nginx conf was generated in "+namespace+": %w", err.Error()) default: // each call to GetNginxConfig takes about 70ms - conf, _ = resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + conf, _ = resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, nginxCrossplanePath) continue } } @@ -402,7 +402,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r // this checks if NGF has established a connection with agent and sent over the first nginx conf Eventually( func() bool { - conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, *ngfImageRepository) + conf, _ := resourceManager.GetNginxConfig(nginxPodName, reconfigNamespace.Name, nginxCrossplanePath) // a default upstream NGF creates defaultUpstream := framework.ExpectedNginxField{ Directive: "upstream", diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index a8e558da6b..6a8174bdb9 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -70,20 +70,21 @@ var ( var ( //go:embed manifests/* - manifests embed.FS - k8sClient client.Client // TODO: are the k8sClient and the resourceManager.k8sClient the same? - resourceManager framework.ResourceManager - portForwardStopCh chan struct{} - portFwdPort int - portFwdHTTPSPort int - timeoutConfig framework.TimeoutConfig - localChartPath string - address string - version string - chartVersion string - clusterInfo framework.ClusterInfo - skipNFRTests bool - logs string + manifests embed.FS + k8sClient client.Client + resourceManager framework.ResourceManager + portForwardStopCh chan struct{} + portFwdPort int + portFwdHTTPSPort int + timeoutConfig framework.TimeoutConfig + localChartPath string + address string + version string + chartVersion string + clusterInfo framework.ClusterInfo + skipNFRTests bool + logs string + nginxCrossplanePath string ) var formatNginxPlusEdgeImagePath = "us-docker.pkg.dev/%s/nginx-gateway-fabric/nginx-plus" @@ -171,6 +172,8 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { version = "edge" } + nginxCrossplanePath = "us-docker.pkg.dev/" + *gkeProject + "/nginx-gateway-fabric" + if !cfg.deploy { return }