MAJOR: crd: add job for custom resource definition handling

CRDs are not properly handled with external tools, Helm and similar
options cannot handle upgrading of custom resource definitions

Author: oktalz

.gitignore

@@ -4,3 +4,4 @@ kubernetes-ingress
 dist/
 .code-generator/
 bin/golangci-lint
+.local/*

crs/definition/backend.yaml renamed to crs/definition/backends.core.haproxy.org.yaml

@@ -0,0 +1,37 @@
+package definition
+
+import _ "embed"
+
+//go:embed defaults.core.haproxy.org.yaml
+var DefaultsV1alpha2 []byte
+
+//go:embed globals.core.haproxy.org.yaml
+var GlobalsV1alpha2 []byte
+
+//go:embed backends.core.haproxy.org.yaml
+var BackendsV1alpha2 []byte
+
+//go:embed upgrade/defaults.core.haproxy.org.yaml
+var DefaultsV1alpha1V1alpha2 []byte
+
+//go:embed upgrade/globals.core.haproxy.org.yaml
+var GlobalsV1alpha1V1alpha2 []byte
+
+//go:embed upgrade/backends.core.haproxy.org.yaml
+var BackendsV1alpha1V1alpha2 []byte
+
+func GetCRDs() map[string][]byte {
+	return map[string][]byte{
+		"defaults.core.haproxy.org": DefaultsV1alpha2,
+		"globals.core.haproxy.org":  GlobalsV1alpha2,
+		"backends.core.haproxy.org": BackendsV1alpha2,
+	}
+}
+
+func GetCRDsUpgrade() map[string][]byte {
+	return map[string][]byte{
+		"defaults.core.haproxy.org": DefaultsV1alpha1V1alpha2,
+		"globals.core.haproxy.org":  GlobalsV1alpha1V1alpha2,
+		"backends.core.haproxy.org": BackendsV1alpha1V1alpha2,
+	}
+}

crs/definition/defaults.yaml renamed to crs/definition/defaults.core.haproxy.org.yaml

@@ -0,0 +1,16 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: haproxy-ingress-crd # each deploymnent should have a unique name, in example we always recreate the custer
+  namespace: haproxy-controller
+spec:
+  template:
+    spec:
+      serviceAccountName: haproxy-kubernetes-ingress-crd
+      containers:
+      - name: haproxy-ingress-crd
+        image: haproxytech/kubernetes-ingress:latest
+        imagePullPolicy: Never
+        command: ["./haproxy-ingress-controller","--job-check-crd"]
+      restartPolicy: Never
+  backoffLimit: 0

crs/definition/embed.go

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+  namespace: haproxy-controller
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - customresourcedefinitions
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+  namespace: haproxy-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: haproxy-kubernetes-ingress-crd
+subjects:
+  - kind: ServiceAccount
+    name: haproxy-kubernetes-ingress-crd
+    namespace: haproxy-controller

crs/definition/global.yaml renamed to crs/definition/globals.core.haproxy.org.yaml

@@ -11,7 +11,6 @@ if [ -n "${CI_ENV}" ]; then
   echo "building image for ingress controller"
   if [ -n "${GITLAB_CI}" ]; then
     echo "haproxytech/kubernetes-ingress image already available from previous stage"
-    #docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
   else
     docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
   fi
@@ -44,13 +43,16 @@ fi
 echo "loading image http-echo in kind"
 kind load docker-image haproxytech/http-echo:latest  --name=$clustername
 
+printf %80s |tr " " "="; echo ""
+echo "Create HAProxy namespace ..."
+kubectl apply -f $DIR/config/0.namespace.yaml
+printf %80s |tr " " "="; echo ""
 echo "Install custom resource definitions ..."
-kubectl apply -f $DIR/../../crs/definition/backend.yaml
-kubectl apply -f $DIR/../../crs/definition/defaults.yaml
-kubectl apply -f $DIR/../../crs/definition/global.yaml
+kubectl apply -f $DIR/../../deploy/tests/config/crd/rbac.yaml
+kubectl apply -f $DIR/../../deploy/tests/config/crd/job-crd.yaml
+kubectl wait --for=condition=complete --timeout=60s job/haproxy-ingress-crd -n haproxy-controller
 
 echo "deploying Ingress Controller ..."
-kubectl apply -f $DIR/config/0.namespace.yaml
 kubectl apply -f $DIR/config/1.rbac.yaml
 kubectl apply -f $DIR/config/2.configmap.yaml
 kubectl apply -f $DIR/config/3.ingress-controller.yaml

crs/definition/upgrade/backend.yaml renamed to crs/definition/upgrade/backends.core.haproxy.org.yaml

@@ -16,8 +16,10 @@ require (
 	github.com/valyala/fasthttp v1.50.0
 	go.uber.org/automaxprocs v1.5.3
 	k8s.io/api v0.25.2
+	k8s.io/apiextensions-apiserver v0.25.2
 	k8s.io/apimachinery v0.25.2
 	k8s.io/client-go v0.25.2
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -58,6 +60,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
+	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
@@ -83,5 +87,4 @@ require (
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )

crs/definition/upgrade/defaults.yaml renamed to crs/definition/upgrade/defaults.core.haproxy.org.yaml

@@ -127,6 +127,8 @@ github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9F
 github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
 github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -186,6 +188,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU=
 github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -266,10 +270,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
-github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
-github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
-github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
+github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
+github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
@@ -457,6 +461,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -509,6 +515,8 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8=
 k8s.io/api v0.25.2/go.mod h1:qP1Rn4sCVFwx/xIhe+we2cwBLTXNcheRyYXwajonhy0=
+k8s.io/apiextensions-apiserver v0.25.2 h1:8uOQX17RE7XL02ngtnh3TgifY7EhekpK+/piwzQNnBo=
+k8s.io/apiextensions-apiserver v0.25.2/go.mod h1:iRwwRDlWPfaHhuBfQ0WMa5skdQfrE18QXJaJvIDLvE8=
 k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs=
 k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA=
 k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo=
@@ -524,5 +532,5 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
 sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

crs/definition/upgrade/global.yaml renamed to crs/definition/upgrade/globals.core.haproxy.org.yaml

@@ -34,6 +34,7 @@ import (
 
 	"github.com/haproxytech/kubernetes-ingress/pkg/annotations"
 	"github.com/haproxytech/kubernetes-ingress/pkg/controller"
+	"github.com/haproxytech/kubernetes-ingress/pkg/job"
 	"github.com/haproxytech/kubernetes-ingress/pkg/k8s"
 	"github.com/haproxytech/kubernetes-ingress/pkg/store"
 	"github.com/haproxytech/kubernetes-ingress/pkg/utils"
@@ -70,6 +71,21 @@ func main() {
 		parser.WriteHelp(os.Stdout)
 		return
 	}
+	if osArgs.JobCheckCRD {
+		logger.Print(IngressControllerInfo)
+		logger.Print(job.IngressControllerCRDUpdater)
+		logger.Infof("HAProxy Ingress Controller CRD Updater %s %s%s", GitTag, GitCommit, GitDirty)
+		logger.Infof("Build from: %s", GitRepo)
+
+		err := job.CRDRefresh(logger, osArgs)
+		if err != nil {
+			logger.Error(err)
+			os.Exit(1) //nolint:gocritic
+		}
+		// exit, this is just a job
+		os.Exit(0)
+	}
+
 	logger.ShowFilename(false)
 	exit := logInfo(logger, osArgs)
 	if exit {

deploy/tests/config/crd/job-crd.yaml

@@ -0,0 +1,103 @@
+// Copyright 2023 HAProxy Technologies LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package job
+
+import (
+	"context"
+
+	"github.com/haproxytech/kubernetes-ingress/crs/definition"
+	"github.com/haproxytech/kubernetes-ingress/pkg/k8s"
+	"github.com/haproxytech/kubernetes-ingress/pkg/utils"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiError "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+)
+
+func CRDRefresh(log utils.Logger, osArgs utils.OSArgs) error {
+	log.Info("checking CRDS")
+	config, err := k8s.GetRestConfig(osArgs)
+	if err != nil {
+		return err
+	}
+
+	// Create a new clientset for the apiextensions API group
+	clientset := apiextensionsclientset.NewForConfigOrDie(config)
+
+	// Check if the CRD exists
+	crds := definition.GetCRDs()
+	crdsUpgrade := definition.GetCRDsUpgrade()
+	for crdName, crdDef := range crds {
+		// CustomResourceDefinition object
+		var crd apiextensionsv1.CustomResourceDefinition
+		err = yaml.Unmarshal(crdDef, &crd)
+		if err != nil {
+			return err
+		}
+		log.Info("")
+		log.Infof("checking CRD %s", crdName)
+
+		existingVersion, err := clientset.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), crdName, metav1.GetOptions{})
+		if err != nil {
+			if !apiError.IsNotFound(err) {
+				return err
+			}
+			log.Infof("CRD %s does not exist", crdName)
+			// Create the CRD
+			_, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crd, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
+			log.Infof("CRD %s created", crdName)
+			continue
+		}
+		log.Infof("CRD %s exists", crdName)
+		versions := existingVersion.Spec.Versions
+		if len(versions) == 2 {
+			log.Infof("CRD %s exists as v1alpha1 and v1alpha2, nothing to do", crdName)
+			continue
+		}
+		// check if we have alpha 2 or we need to upgrade for alpha2
+		crd.ObjectMeta.ResourceVersion = existingVersion.ObjectMeta.ResourceVersion
+		if versions[0].Name == "v1alpha2" {
+			log.Infof("CRD %s exists as v1alpha2, nothing to do", crdName)
+			continue
+		}
+		err = yaml.Unmarshal(crdsUpgrade[crdName], &crd)
+		if err != nil {
+			return err
+		}
+		// Upgrade the CRDl
+		_, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Update(context.Background(), &crd, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Info("")
+	log.Info("CRD update done")
+	return nil
+}
+
+// IngressControllerCRDUpdater console pretty print
+const IngressControllerCRDUpdater = `
+  ____ ____  ____    _   _           _       _
+ / ___|  _ \|  _ \  | | | |_ __   __| | __ _| |_ ___ _ __
+| |   | |_) | | | | | | | | '_ \ / _` + "`" + ` |/ _` + "`" + ` | __/ _ \ '__|
+| |___|  _ <| |_| | | |_| | |_) | (_| | (_| | ||  __/ |
+ \____|_| \_\____/   \___/| .__/ \__,_|\__,_|\__\___|_|
+                          |_|
+
+`