add support for version & adjust flags (kubernetes-sigs#1506)

* code adjustments

* make the error logging concise

Author: M00nF1sh

Dockerfile

@@ -12,12 +12,16 @@ RUN GOPROXY=direct go mod download
 FROM base AS build
 ARG TARGETOS
 ARG TARGETARCH
+ENV VERSION_PKG=sigs.k8s.io/aws-load-balancer-controller/pkg/version
 RUN --mount=type=bind,target=. \
     --mount=type=cache,target=/root/.cache/go-build \
+    GIT_VERSION=$(git describe --tags --dirty --always) && \
+    GIT_COMMIT=$(git rev-parse HEAD) && \
+    BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) && \
     GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on \
     CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2" \
     CGO_LDFLAGS="-Wl,-z,relro,-z,now" \
-    go build -ldflags="-s -w" -buildmode=pie -a -o /out/controller main.go
+    go build -ldflags="-s -w -X ${VERSION_PKG}.GitVersion=${GIT_VERSION} -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" -buildmode=pie -a -o /out/controller main.go
 
 FROM amazonlinux:2 as bin-unix
 COPY --from=build /out/controller /controller

controllers/elbv2/targetgroupbinding_controller.go

@@ -50,7 +50,7 @@ func NewTargetGroupBindingReconciler(k8sClient client.Client, finalizerManager k
 		tgbResourceManager: tgbResourceManager,
 		logger:             logger,
 
-		maxConcurrentReconciles: config.TargetgroupBindingMaxConcurrentReconciles,
+		maxConcurrentReconciles: config.TargetGroupBindingMaxConcurrentReconciles,
 	}
 }
 

main.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"github.com/spf13/pflag"
 	zapraw "go.uber.org/zap"
-	"k8s.io/apimachinery/pkg/runtime"
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"os"
@@ -31,10 +31,12 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
-	inject "sigs.k8s.io/aws-load-balancer-controller/pkg/inject"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/inject"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/version"
 	corewebhook "sigs.k8s.io/aws-load-balancer-controller/webhooks/core"
 	elbv2webhook "sigs.k8s.io/aws-load-balancer-controller/webhooks/elbv2"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -44,7 +46,7 @@ import (
 )
 
 var (
-	scheme   = runtime.NewScheme()
+	scheme   = k8sruntime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
 )
 
@@ -56,41 +58,31 @@ func init() {
 }
 
 func main() {
-	awsCloudConfig := aws.CloudConfig{ThrottleConfig: throttle.NewDefaultServiceOperationsThrottleConfig()}
-	injectConfig := inject.Config{}
-	controllerConfig := config.ControllerConfig{}
-
-	fs := pflag.NewFlagSet("", pflag.ExitOnError)
-	awsCloudConfig.BindFlags(fs)
-	injectConfig.BindFlags(fs)
-	controllerConfig.BindFlags(fs)
-
-	if err := fs.Parse(os.Args); err != nil {
-		setupLog.Error(err, "invalid flags")
-		os.Exit(1)
-	}
-	if err := controllerConfig.Validate(); err != nil {
-		setupLog.Error(err, "Failed to validate controller configuration")
+	setLogLevel("info")
+	setupLog.Info("version",
+		"GitVersion", version.GitVersion,
+		"GitCommit", version.GitCommit,
+		"BuildDate", version.BuildDate,
+	)
+	controllerCFG, err := loadControllerConfig()
+	if err != nil {
+		setupLog.Error(err, "unable to load controller config")
 		os.Exit(1)
 	}
+	setLogLevel(controllerCFG.LogLevel)
 
-	logLevel := zapraw.NewAtomicLevelAt(0)
-	if controllerConfig.LogLevel == "debug" {
-		logLevel = zapraw.NewAtomicLevelAt(-1)
-	}
-	ctrl.SetLogger(zap.New(zap.UseDevMode(false), zap.Level(&logLevel)))
-
-	cloud, err := aws.NewCloud(awsCloudConfig, metrics.Registry)
+	cloud, err := aws.NewCloud(controllerCFG.AWSConfig, metrics.Registry)
 	if err != nil {
-		setupLog.Error(err, "Unable to initialize AWS cloud")
+		setupLog.Error(err, "unable to initialize AWS cloud")
 		os.Exit(1)
 	}
-	restCfg, err := config.BuildRestConfig(controllerConfig.RuntimeConfig)
-	rtOpts := config.BuildRuntimeOptions(controllerConfig.RuntimeConfig, scheme)
+	restCFG, err := config.BuildRestConfig(controllerCFG.RuntimeConfig)
 	if err != nil {
-		setupLog.Error(err, "Unable to build REST config")
+		setupLog.Error(err, "unable to build REST config")
+		os.Exit(1)
 	}
-	mgr, err := ctrl.NewManager(restCfg, rtOpts)
+	rtOpts := config.BuildRuntimeOptions(controllerCFG.RuntimeConfig, scheme)
+	mgr, err := ctrl.NewManager(restCFG, rtOpts)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
@@ -101,18 +93,18 @@ func main() {
 	nodeENIResolver := networking.NewDefaultNodeENIInfoResolver(cloud.EC2(), ctrl.Log)
 	sgManager := networking.NewDefaultSecurityGroupManager(cloud.EC2(), ctrl.Log)
 	sgReconciler := networking.NewDefaultSecurityGroupReconciler(sgManager, ctrl.Log)
-	subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), controllerConfig.ClusterName, ctrl.Log.WithName("subnets-resolver"))
+	subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log.WithName("subnets-resolver"))
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
-		podENIResolver, nodeENIResolver, sgManager, sgReconciler, cloud.VpcID(), controllerConfig.ClusterName, ctrl.Log)
+		podENIResolver, nodeENIResolver, sgManager, sgReconciler, cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log)
 
 	ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"),
 		finalizerManager, sgManager, sgReconciler, subnetResolver,
-		controllerConfig, ctrl.Log.WithName("controllers").WithName("ingress"))
+		controllerCFG, ctrl.Log.WithName("controllers").WithName("ingress"))
 	svcReconciler := service.NewServiceReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("service"),
 		finalizerManager, sgManager, sgReconciler, subnetResolver,
-		controllerConfig, ctrl.Log.WithName("controllers").WithName("service"))
+		controllerCFG, ctrl.Log.WithName("controllers").WithName("service"))
 	tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), finalizerManager, tgbResManager,
-		controllerConfig, ctrl.Log.WithName("controllers").WithName("targetGroupBinding"))
+		controllerCFG, ctrl.Log.WithName("controllers").WithName("targetGroupBinding"))
 	ctx := context.Background()
 	if err = ingGroupReconciler.SetupWithManager(ctx, mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Ingress")
@@ -127,7 +119,8 @@ func main() {
 		os.Exit(1)
 	}
 
-	podReadinessGateInjector := inject.NewPodReadinessGate(injectConfig, mgr.GetClient(), ctrl.Log.WithName("pod-readiness-gate-injector"))
+	podReadinessGateInjector := inject.NewPodReadinessGate(controllerCFG.PodWebhookConfig,
+		mgr.GetClient(), ctrl.Log.WithName("pod-readiness-gate-injector"))
 	corewebhook.NewPodMutator(podReadinessGateInjector).SetupWithManager(mgr)
 	elbv2webhook.NewTargetGroupBindingMutator(cloud.ELBV2(), ctrl.Log).SetupWithManager(mgr)
 	elbv2webhook.NewTargetGroupBindingValidator(ctrl.Log).SetupWithManager(mgr)
@@ -139,3 +132,41 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// loadControllerConfig loads the controller configuration.
+func loadControllerConfig() (config.ControllerConfig, error) {
+	defaultAWSThrottleCFG := throttle.NewDefaultServiceOperationsThrottleConfig()
+	controllerCFG := config.ControllerConfig{
+		AWSConfig: aws.CloudConfig{ThrottleConfig: defaultAWSThrottleCFG},
+	}
+
+	fs := pflag.NewFlagSet("", pflag.ExitOnError)
+	controllerCFG.BindFlags(fs)
+
+	if err := fs.Parse(os.Args); err != nil {
+		return config.ControllerConfig{}, err
+	}
+
+	if err := controllerCFG.Validate(); err != nil {
+		return config.ControllerConfig{}, err
+	}
+	return controllerCFG, nil
+}
+
+// setLogLevel sets the log level of controller.
+func setLogLevel(logLevel string) {
+	var zapLevel zapraw.AtomicLevel
+	switch logLevel {
+	case "info":
+		zapLevel = zapraw.NewAtomicLevelAt(zapraw.InfoLevel)
+	case "debug":
+		zapLevel = zapraw.NewAtomicLevelAt(zapraw.DebugLevel)
+	default:
+		zapLevel = zapraw.NewAtomicLevelAt(zapraw.DebugLevel)
+	}
+
+	logger := zap.New(zap.UseDevMode(false),
+		zap.Level(zapLevel),
+		zap.StacktraceLevel(zapraw.NewAtomicLevelAt(zapraw.FatalLevel)))
+	ctrl.SetLogger(runtime.NewConciseLogger(logger))
+}

pkg/config/controller_config.go

@@ -1,15 +1,17 @@
 package config
 
 import (
-	"fmt"
+	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/inject"
 )
 
 const (
 	flagLogLevel                                  = "log-level"
 	flagK8sClusterName                            = "cluster-name"
 	flagServiceMaxConcurrentReconciles            = "service-max-concurrent-reconciles"
-	flagTargetgroupBindingMaxConcurrentReconciles = "targetgroupbinding-max-concurrent-reconciles"
+	flagTargetGroupBindingMaxConcurrentReconciles = "targetgroupbinding-max-concurrent-reconciles"
 	defaultLogLevel                               = "info"
 	defaultMaxConcurrentReconciles                = 3
 )
@@ -20,16 +22,21 @@ type ControllerConfig struct {
 	LogLevel string
 	// Name of the Kubernetes cluster
 	ClusterName string
+	// Configurations for AWS.
+	AWSConfig aws.CloudConfig
+	// Configurations for the Controller Runtime
+	RuntimeConfig RuntimeConfig
+	// Configurations for Pod inject webhook
+	PodWebhookConfig inject.Config
 	// Configurations for the Ingress controller
 	IngressConfig IngressConfig
 	// Configurations for Addons feature
 	AddonsConfig AddonsConfig
-	// Configurations for the Controller Runtime
-	RuntimeConfig RuntimeConfig
+
 	// Max concurrent reconcile loops for Service objects
 	ServiceMaxConcurrentReconciles int
 	// Max concurrent reconcile loops for TargetGroupBinding objects
-	TargetgroupBindingMaxConcurrentReconciles int
+	TargetGroupBindingMaxConcurrentReconciles int
 }
 
 // BindFlags binds the command line flags to the fields in the config object
@@ -39,18 +46,21 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&cfg.ClusterName, flagK8sClusterName, "", "Kubernetes cluster name")
 	fs.IntVar(&cfg.ServiceMaxConcurrentReconciles, flagServiceMaxConcurrentReconciles, defaultMaxConcurrentReconciles,
 		"Maximum number of concurrently running reconcile loops for service")
-	fs.IntVar(&cfg.TargetgroupBindingMaxConcurrentReconciles, flagTargetgroupBindingMaxConcurrentReconciles, defaultMaxConcurrentReconciles,
-		"Maximum number of concurrently running reconcile loops for targetgroup binding")
+	fs.IntVar(&cfg.TargetGroupBindingMaxConcurrentReconciles, flagTargetGroupBindingMaxConcurrentReconciles, defaultMaxConcurrentReconciles,
+		"Maximum number of concurrently running reconcile loops for targetGroupBinding")
+
+	cfg.AWSConfig.BindFlags(fs)
+	cfg.RuntimeConfig.BindFlags(fs)
 
+	cfg.PodWebhookConfig.BindFlags(fs)
 	cfg.IngressConfig.BindFlags(fs)
 	cfg.AddonsConfig.BindFlags(fs)
-	cfg.RuntimeConfig.BindFlags(fs)
 }
 
 // Validate the controller configuration
 func (cfg *ControllerConfig) Validate() error {
 	if len(cfg.ClusterName) == 0 {
-		return fmt.Errorf("Kubernetes cluster name must be specified")
+		return errors.New("kubernetes cluster name must be specified")
 	}
 	return nil
 }

pkg/config/runtime_config.go

@@ -6,7 +6,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"time"
 )
@@ -19,10 +18,8 @@ const (
 	flagLeaderElectionNamespace = "leader-election-namespace"
 	flagWatchNamespace          = "watch-namespace"
 	flagSyncPeriod              = "sync-period"
-	flagMaster                  = "master"
 	flagKubeconfig              = "kubeconfig"
 
-	defaultMaster                  = ""
 	defaultKubeconfig              = ""
 	defaultLeaderElectionID        = "aws-load-balancer-controller-leader"
 	defaultLeaderElectionNamespace = ""
@@ -53,8 +50,6 @@ type RuntimeConfig struct {
 
 // BindFlags binds the command line flags to the fields in the config object
 func (c *RuntimeConfig) BindFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&c.APIServer, flagMaster, defaultMaster,
-		"The address of the Kubernetes API server.")
 	fs.StringVar(&c.KubeConfig, flagKubeconfig, defaultKubeconfig,
 		"Path to the kubeconfig file containing authorization and API server information.")
 	fs.StringVar(&c.MetricsBindAddress, flagMetricsBindAddr, defaultMetricsAddr,
@@ -74,27 +69,23 @@ func (c *RuntimeConfig) BindFlags(fs *pflag.FlagSet) {
 		"Period at which the controller forces the repopulation of its local object stores.")
 }
 
-func buildRestConfig(masterURL, kubeconfigPath string) (*rest.Config, error) {
-	if kubeconfigPath == "" && masterURL == "" {
-		kubeconfig, err := rest.InClusterConfig()
-		if err == nil {
-			return kubeconfig, nil
-		}
-	}
-	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterURL}}).ClientConfig()
-}
-
 // BuildRestConfig builds the REST config for the controller runtime
 func BuildRestConfig(rtCfg RuntimeConfig) (*rest.Config, error) {
-	restCfg, err := buildRestConfig(rtCfg.APIServer, rtCfg.KubeConfig)
+	var restCFG *rest.Config
+	var err error
+	if rtCfg.KubeConfig == "" {
+		restCFG, err = rest.InClusterConfig()
+	} else {
+		restCFG, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			&clientcmd.ClientConfigLoadingRules{ExplicitPath: rtCfg.KubeConfig}, &clientcmd.ConfigOverrides{}).ClientConfig()
+	}
 	if err != nil {
 		return nil, err
 	}
-	restCfg.QPS = defaultQPS
-	restCfg.Burst = defaultBurst
-	return restCfg, nil
+
+	restCFG.QPS = defaultQPS
+	restCFG.Burst = defaultBurst
+	return restCFG, nil
 }
 
 // BuildRuntimeOptions builds the options for the controller runtime based on config

pkg/runtime/concise_logger.go

@@ -0,0 +1,42 @@
+package runtime
+
+import "github.com/go-logr/logr"
+
+// NewConciseLogger constructs new conciseLogger
+func NewConciseLogger(logger logr.Logger) *conciseLogger {
+	return &conciseLogger{
+		Logger: logger,
+	}
+}
+
+var _ logr.Logger = &conciseLogger{}
+
+// conciseLogger will log concise Error messages.
+// We have used github.com/pkg/errors extensively, when logged with zap logger, a full stacktrace is logged but it's usually unhelpful due to go routine usage.
+// this conciseLogger will wrap the error inside a conciseError, so that only necessary error message is logged.
+type conciseLogger struct {
+	logr.Logger
+}
+
+func (r *conciseLogger) Error(err error, msg string, keysAndValues ...interface{}) {
+	r.Logger.Error(&conciseError{err: err}, msg, keysAndValues...)
+}
+
+func (r *conciseLogger) WithValues(keysAndValues ...interface{}) logr.Logger {
+	return NewConciseLogger(r.Logger.WithValues(keysAndValues...))
+}
+
+func (r *conciseLogger) WithName(name string) logr.Logger {
+	return NewConciseLogger(r.Logger.WithName(name))
+}
+
+var _ error = &conciseError{}
+
+// conciseError will only contain concise error message.
+type conciseError struct {
+	err error
+}
+
+func (e *conciseError) Error() string {
+	return e.err.Error()
+}