cnfcert-203: check csi certified (#570)

* cnfcert-203: check csi certified

* cnfcert-203: check csi certified

* cnfcert-203: check csi certified init test

* cnfcert-203: fix the linter

* cnfcert-203: command updates

* cnfcert-203: remove the sh file

* cnfcert-203: fux linter

* cnfcert-203: update the command timeout

* cnfcert-203: updates for javeir comments

* cnfcert-203: updates for jUN comments

* cnfcert-203: updates for jUN comments

* cnfcert-203: updates the function verrables

* adding cert-check tool

* unmarshaling embeeded file to match csi driver

* renamed csi-mapping.json file

* Improving operator check to include ocp version

* linter

* update the oc version

* update the os line for the linter

* update the split

* update the split

* remove the html file

* remove the not needed comments

* check all the operators on the cluster

* check all the operator that have certified-operators in org

* remove not relevant files

* remove not relevant calls of csi

* remove not relevant calls of csi

* fix the conflict

* fix the linet

* fix the unit test

* fix the unit test

* fix the unit test

* fix the unit test

* few updates

* few updates

* few updates

* few updates

* few updates

* few updates

* few updates

* updating with jun PR

Co-authored-by: David Elie-Dit-Cosaque <[email protected]>
Co-authored-by: edcdavid <[email protected]>

Author: aabughosh

README.md

@@ -543,3 +543,9 @@ For example:
 ```shell script
 TNF_DEFAULT_BUFFER_SIZE=32768 ./run-cnf-suites.sh -f diagnostic
 ```
+
+# Testing certified operator
+to test if operator certified need to label it with this command
+```shell script
+oc label csv <name> -n <namespace> "test-network-function.com/operator=target" 
+```

internal/api/api.go

@@ -195,8 +195,8 @@ func CreateContainerCatalogQueryURL(id configsections.ContainerImageIdentifier)
 
 // IsOperatorCertified get operator bundle by package name and check if package details is present
 // If present then returns `true` as certified operators.
-func (api CertAPIClient) IsOperatorCertified(org, packageName string) (bool, error) {
-	imageID, err := api.GetOperatorBundleIDByPackageName(org, packageName)
+func (api CertAPIClient) IsOperatorCertified(org, packageName, version string) (bool, error) {
+	imageID, err := api.GetOperatorBundleIDByPackageName(org, packageName, version)
 	if err == nil {
 		if imageID == "" {
 			return false, nil
@@ -219,9 +219,14 @@ func (api CertAPIClient) GetImageByID(id string) (string, error) {
 
 // GetOperatorBundleIDByPackageName get published operator bundle Id by organization and package name.
 // Returns (ImageID, error).
-func (api CertAPIClient) GetOperatorBundleIDByPackageName(org, name string) (string, error) {
+func (api CertAPIClient) GetOperatorBundleIDByPackageName(org, name, vsersion string) (string, error) {
 	var imageID string
-	url := fmt.Sprintf("%s/bundles?page_size=1&organization=%s&package=%s", apiOperatorCatalogExternalBaseEndPoint, org, name)
+	url := ""
+	if vsersion != "" {
+		url = fmt.Sprintf("%s/bundles?page_size=1&filter=organization==%s;csv_name==%s;ocp_version==%s", apiOperatorCatalogExternalBaseEndPoint, org, name, vsersion)
+	} else {
+		url = fmt.Sprintf("%s/bundles?page_size=1&filter=organization==%s;csv_name==%s", apiOperatorCatalogExternalBaseEndPoint, org, name)
+	}
 	responseData, err := api.getRequest(url)
 	if err == nil {
 		imageID, err = api.getIDFromResponse(responseData)

internal/api/api_test.go

@@ -22,6 +22,7 @@ const (
 	unKnownRepository  = "wrong_repo"
 	unKnownImageName   = "wrong_id"
 	unknownPackageName = "unknownPackage"
+	version            = "4.8"
 	jsonResponseFound  = `{
 		"data": [
 		  {
@@ -502,11 +503,12 @@ var (
 		expectedResult bool
 		responseData   string
 		responseStatus int
+		version        string
 	}{
 		{packageName: packageName, org: redHatOrg, expectedError: nil, id: "", expectedResult: true,
-			responseData: jsonResponseFound, responseStatus: http.StatusAccepted},
+			responseData: jsonResponseFound, responseStatus: http.StatusAccepted, version: version},
 		{packageName: unknownPackageName, org: marketPlaceOrg, expectedError: nil, id: "", expectedResult: false,
-			responseData: jsonResponseNotFound, responseStatus: http.StatusNotFound},
+			responseData: jsonResponseNotFound, responseStatus: http.StatusNotFound, version: version},
 	}
 
 	containerQueryURLTestCases = []struct {
@@ -552,7 +554,7 @@ func TestApiClient_GetContainerCatalogEntry(t *testing.T) {
 func TestApiClient_IsOperatorCertified(t *testing.T) {
 	for _, c := range operatorTestCases {
 		GetDoFunc = getDoFunc(c.responseData, c.responseStatus) //nolint:bodyclose
-		result, err := client.IsOperatorCertified(c.org, c.packageName)
+		result, err := client.IsOperatorCertified(c.org, c.packageName, c.version)
 		assert.Equal(t, c.expectedResult, result)
 		assert.Equal(t, c.expectedError, err)
 	}

pkg/config/autodiscover/autodiscover.go

@@ -37,7 +37,7 @@ const (
 	anyLabelValue    = ""
 	ocCommand        = "oc get %s -n %s -o json -l %s"
 	ocAllCommand     = "oc get %s -A -o json -l %s"
-	ocCommandTimeOut = time.Second * 10
+	ocCommandTimeOut = time.Second * 15
 )
 
 var (

pkg/config/autodiscover/autodiscover_targets.go

@@ -83,7 +83,7 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa
 	for _, csv := range csvs.Items {
 		if ns[csv.Metadata.Namespace] {
 			csv := csv
-			target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csv))
+			target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csv, false))
 		}
 	}
 	dps := FindTestPodSetsByLabel(labels, string(configsections.Deployment))
@@ -216,7 +216,7 @@ func buildPodUnderTest(pr *PodResource) (podUnderTest *configsections.Pod) {
 }
 
 // buildOperatorFromCSVResource builds a single `configsections.Operator` from a CSVResource
-func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) {
+func buildOperatorFromCSVResource(csv *CSVResource, istest bool) (op configsections.Operator) {
 	var err error
 	op.Name = csv.Metadata.Name
 	op.Namespace = csv.Metadata.Namespace
@@ -237,6 +237,10 @@ func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator)
 	} else {
 		op.SubscriptionName = subscriptionName[0]
 	}
+	if !istest {
+		op.Packag, op.Org, op.Version = csv.PackOrgVersion(op.Name)
+	}
+
 	return op
 }
 

pkg/config/autodiscover/csv_info.go

@@ -18,6 +18,7 @@ package autodiscover
 
 import (
 	"fmt"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
 
@@ -61,6 +62,17 @@ func (csv *CSVResource) GetAnnotationValue(annotationKey string, v interface{})
 	}
 	return err
 }
+func (csv *CSVResource) PackOrgVersion(subscription string) (org, packag, version string) {
+	ocCmd := fmt.Sprintf("oc get subscriptions.operators.coreos.com -A -o go-template='{{range .items}}{{if eq .status.installedCSV %q}}{{.spec.source}} {{.status.currentCSV}}{{end}}{{end}}'", subscription)
+	out := execCommandOutput(ocCmd)
+	orgNameVer := strings.Split(out, " ")
+	org = orgNameVer[0]
+	nameVersion := strings.SplitN(orgNameVer[1], ".", 2) //nolint:gomnd // ok
+	packag = orgNameVer[1]
+	version = nameVersion[1]
+
+	return packag, org, version
+}
 
 func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error) error {
 	return fmt.Errorf("error (%s) attempting to unmarshal value of annotation '%s' on CSV '%s/%s'",

pkg/config/autodiscover/operator_test.go

@@ -24,7 +24,7 @@ import (
 
 func TestBuildOperatorFromCSVResource(t *testing.T) {
 	csvResource := loadCSVResource(csvFilePath)
-	operator := buildOperatorFromCSVResource(&csvResource)
+	operator := buildOperatorFromCSVResource(&csvResource, true)
 
 	assert.Equal(t, "CSVNamespace", operator.Namespace)
 	assert.Equal(t, "CSVName", operator.Name)

pkg/config/configsections/common.go

@@ -38,6 +38,12 @@ type Operator struct {
 
 	// Subscription name is required field, Name of used subscription.
 	SubscriptionName string `yaml:"subscriptionName" json:"subscriptionName"`
+
+	Packag string `yaml:"packag" json:"packag"`
+
+	Org string `yaml:"Org" json:"Org"`
+
+	Version string `yaml:"Version" json:"Version"`
 }
 
 // Namespace struct defines namespace properties
@@ -91,6 +97,7 @@ type TestTarget struct {
 	ExcludeContainersFromConnectivityTests []ContainerIdentifier `yaml:"excludeContainersFromConnectivityTests" json:"excludeContainersFromConnectivityTests"`
 	// Operator is the list of operator objects that needs to be tested.
 	Operators []Operator `yaml:"operators,omitempty"  json:"operators,omitempty"`
+	//
 	// Node list
 	Nodes map[string]Node `yaml:"Nodes"  json:"Nodes"`
 }

test-network-function/certification/suite.go

@@ -18,6 +18,7 @@ package certification
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
@@ -26,18 +27,33 @@ import (
 	configpkg "github.com/test-network-function/test-network-function/pkg/config"
 	"github.com/test-network-function/test-network-function/pkg/config/configsections"
 	"github.com/test-network-function/test-network-function/pkg/tnf"
+	"github.com/test-network-function/test-network-function/pkg/tnf/interactive"
 	"github.com/test-network-function/test-network-function/pkg/tnf/testcases"
+	"github.com/test-network-function/test-network-function/pkg/utils"
 	"github.com/test-network-function/test-network-function/test-network-function/common"
 	"github.com/test-network-function/test-network-function/test-network-function/identifiers"
 	"github.com/test-network-function/test-network-function/test-network-function/results"
 )
 
 const (
 	// timeout for eventually call
-	apiRequestTimeout = 30 * time.Second
+	apiRequestTimeout           = 40 * time.Second
+	expectersVerboseModeEnabled = false
+	CertifiedOperator           = "certified-operators"
+	outMinikubeVersion          = "null"
 )
 
-var certAPIClient api.CertAPIClient
+var (
+	ocpVersionCommand = "oc version -o json | jq '.openshiftVersion'"
+
+	execCommandOutput = func(command string) string {
+		return utils.ExecuteCommandAndValidate(command, apiRequestTimeout, interactive.GetContext(expectersVerboseModeEnabled), func() {
+			log.Error("can't run command: ", command)
+		})
+	}
+
+	certAPIClient api.CertAPIClient
+)
 
 var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() {
 	conf, _ := ginkgo.GinkgoConfiguration()
@@ -51,7 +67,7 @@ var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() {
 		ginkgo.AfterEach(env.CloseLocalShellContext)
 
 		testContainerCertificationStatus()
-		testOperatorCertificationStatus()
+		testAllOperatorCertified(env)
 	}
 })
 
@@ -63,9 +79,9 @@ func getContainerCertificationRequestFunction(id configsections.ContainerImageId
 }
 
 // getOperatorCertificationRequestFunction returns function that will try to get the certification status (OCP) for an operator.
-func getOperatorCertificationRequestFunction(organization, operatorName string) func() (interface{}, error) {
+func getOperatorCertificationRequestFunction(organization, operatorName, ocpversion string) func() (interface{}, error) {
 	return func() (interface{}, error) {
-		return certAPIClient.IsOperatorCertified(organization, operatorName)
+		return certAPIClient.IsOperatorCertified(organization, operatorName, ocpversion)
 	}
 }
 
@@ -141,43 +157,49 @@ func testContainerCertificationStatus() {
 	})
 }
 
-func testOperatorCertificationStatus() {
+func testAllOperatorCertified(env *configpkg.TestEnvironment) {
 	testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsCertifiedIdentifier)
 	ginkgo.It(testID, ginkgo.Label(testID), func() {
-		operatorsToQuery := configpkg.GetTestEnvironment().Config.CertifiedOperatorInfo
+		operatorsToQuery := env.OperatorsUnderTest
 
 		if len(operatorsToQuery) == 0 {
-			ginkgo.Skip("No operators to check configured in tnf_config.yml")
+			ginkgo.Skip("No operators to check configured ")
 		}
 
 		ginkgo.By(fmt.Sprintf("Verify operator as certified. Number of operators to check: %d", len(operatorsToQuery)))
-		if len(operatorsToQuery) > 0 {
-			certAPIClient = api.NewHTTPClient()
-			failedOperators := []configsections.CertifiedOperatorRequestInfo{}
-			allOperatorsToQueryEmpty := true
-			for _, operator := range operatorsToQuery {
-				if operator.Name == "" || operator.Organization == "" {
-					tnf.ClaimFilePrintf("Operator name = \"%s\" or organization = \"%s\" is missing, skipping this operator to query", operator.Name, operator.Organization)
-					continue
-				}
-				allOperatorsToQueryEmpty = false
-				ginkgo.By(fmt.Sprintf("Should eventually be verified as certified (operator %s/%s)", operator.Organization, operator.Name))
-				isCertified := waitForCertificationRequestToSuccess(getOperatorCertificationRequestFunction(operator.Organization, operator.Name), apiRequestTimeout).(bool)
+		testFailed := false
+		for _, op := range operatorsToQuery {
+			ocpversion := GetOcpVersion()
+			pack := op.Name
+			org := op.Org
+			if org == CertifiedOperator {
+				isCertified := waitForCertificationRequestToSuccess(getOperatorCertificationRequestFunction(org, pack, ocpversion), apiRequestTimeout).(bool)
 				if !isCertified {
-					tnf.ClaimFilePrintf("Operator %s (organization %s) failed to be certified.", operator.Name, operator.Organization)
-					failedOperators = append(failedOperators, operator)
+					testFailed = true
+					log.Info(fmt.Sprintf("Operator %s (organization %s) not certified for Openshift %s .", pack, org, ocpversion))
+					tnf.ClaimFilePrintf("Operator %s (organization %s) failed to be certified for Openshift %s", pack, org, ocpversion)
 				} else {
-					log.Info(fmt.Sprintf("Operator %s (organization %s) certified OK.", operator.Name, operator.Organization))
+					log.Info(fmt.Sprintf("Operator %s (organization %s) certified OK.", pack, org))
 				}
+			} else {
+				testFailed = true
+				tnf.ClaimFilePrintf("Operator %s is not certified (needs to be part of the operator-certified organization in the catalog)", op.Packag)
 			}
-			if allOperatorsToQueryEmpty {
-				ginkgo.Skip("No operators to check because either operator name or organization is empty for all operators in tnf_config.yml")
-			}
-
-			if n := len(failedOperators); n > 0 {
-				log.Warnf("Operators that failed to be certified: %+v", failedOperators)
-				ginkgo.Fail(fmt.Sprintf("%d operators failed to be certified.", n))
-			}
+		}
+		if testFailed {
+			ginkgo.Skip("At least one  operator was not certified to run on this version of openshift. Check Claim.json file for details.")
 		}
 	})
 }
+
+func GetOcpVersion() string {
+	ocCmd := ocpVersionCommand
+	ocVersion := execCommandOutput(ocCmd)
+	if ocVersion != outMinikubeVersion {
+		nums := strings.Split(strings.ReplaceAll(ocVersion, "\"", ""), ".")
+		ocVersion = nums[0] + "." + nums[1]
+	} else {
+		ocVersion = ""
+	}
+	return ocVersion
+}