Merge pull request kubernetes-sigs#4499 from stmcginnis/no-docker-testing

🌱  Use docker API instead of CLI in test framework

Author: k8s-ci-robot

test/framework/bootstrap/kind_util.go

@@ -25,8 +25,8 @@ import (
 	"github.com/pkg/errors"
 
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	"sigs.k8s.io/cluster-api/test/framework/exec"
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
 	kind "sigs.k8s.io/kind/pkg/cluster"
 	kindnodes "sigs.k8s.io/kind/pkg/cluster/nodes"
 	kindnodesutils "sigs.k8s.io/kind/pkg/cluster/nodeutils"
@@ -149,11 +149,13 @@ func loadImage(ctx context.Context, cluster, image string) error {
 // copied from kind https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/cmd/kind/load/docker-image/docker-image.go#L168
 // save saves image to dest, as in `docker save`.
 func save(ctx context.Context, image, dest string) error {
-	sout, serr, err := exec.NewCommand(
-		exec.WithCommand("docker"),
-		exec.WithArgs("save", "-o", dest, image),
-	).Run(ctx)
-	return errors.Wrapf(err, "stdout: %q, stderr: %q", string(sout), string(serr))
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to get Docker runtime client")
+	}
+
+	err = containerRuntime.SaveContainerImage(ctx, image, dest)
+	return errors.Wrapf(err, "error saving image %q to %q", image, dest)
 }
 
 // copied from kind https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/cmd/kind/load/docker-image/docker-image.go#L158

test/framework/cluster_proxy.go

@@ -24,7 +24,6 @@ import (
 	"os"
 	"path"
 	goruntime "runtime"
-	"strings"
 
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
@@ -37,6 +36,7 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
 	"sigs.k8s.io/cluster-api/test/framework/exec"
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -314,7 +314,10 @@ func (p *clusterProxy) isDockerCluster(ctx context.Context, namespace string, na
 }
 
 func (p *clusterProxy) fixConfig(ctx context.Context, name string, config *api.Config) {
-	port, err := findLoadBalancerPort(ctx, name)
+	containerRuntime, err := container.NewDockerClient()
+	Expect(err).ToNot(HaveOccurred(), "Failed to get Docker runtime client")
+
+	port, err := containerRuntime.GetHostPort(ctx, name, "6443/tcp")
 	Expect(err).ToNot(HaveOccurred(), "Failed to get load balancer port")
 
 	controlPlaneURL := &url.URL{
@@ -325,21 +328,6 @@ func (p *clusterProxy) fixConfig(ctx context.Context, name string, config *api.C
 	config.Clusters[currentCluster].Server = controlPlaneURL.String()
 }
 
-func findLoadBalancerPort(ctx context.Context, name string) (string, error) {
-	loadBalancerName := name + "-lb"
-	portFormat := `{{index (index (index .NetworkSettings.Ports "6443/tcp") 0) "HostPort"}}`
-	getPathCmd := exec.NewCommand(
-		exec.WithCommand("docker"),
-		exec.WithArgs("inspect", loadBalancerName, "--format", portFormat),
-	)
-	stdout, _, err := getPathCmd.Run(ctx)
-	if err != nil {
-		return "", err
-	}
-
-	return strings.TrimSpace(string(stdout)), nil
-}
-
 // Dispose clusterProxy internal resources (the operation does not affects the Kubernetes cluster).
 func (p *clusterProxy) Dispose(ctx context.Context) {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose")

test/framework/docker_logcollector.go

@@ -27,9 +27,9 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/kind/pkg/errors"
-	"sigs.k8s.io/kind/pkg/exec"
 )
 
 // DockerLogCollector collect logs from a CAPD workload cluster.
@@ -47,30 +47,35 @@ func machineContainerName(cluster, machine string) string {
 
 func (k DockerLogCollector) CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error {
 	containerName := machineContainerName(m.Spec.ClusterName, m.Name)
-	return k.collectLogsFromNode(outputPath, containerName)
+	return k.collectLogsFromNode(ctx, outputPath, containerName)
 }
 
 func (k DockerLogCollector) CollectMachinePoolLog(ctx context.Context, managementClusterClient client.Client, m *expv1.MachinePool, outputPath string) error {
 	var errs []error
 	for _, instance := range m.Status.NodeRefs {
 		containerName := machineContainerName(m.Spec.ClusterName, instance.Name)
-		if err := k.collectLogsFromNode(filepath.Join(outputPath, instance.Name), containerName); err != nil {
+		if err := k.collectLogsFromNode(ctx, filepath.Join(outputPath, instance.Name), containerName); err != nil {
 			// collecting logs is best effort so we proceed to the next instance even if we encounter an error.
 			errs = append(errs, err)
 		}
 	}
 	return kerrors.NewAggregate(errs)
 }
 
-func (k DockerLogCollector) collectLogsFromNode(outputPath string, containerName string) error {
+func (k DockerLogCollector) collectLogsFromNode(ctx context.Context, outputPath string, containerName string) error {
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return errors.Wrap(err, "Failed to collect logs from node")
+	}
+
 	execToPathFn := func(outputFileName, command string, args ...string) func() error {
 		return func() error {
 			f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
 			if err != nil {
 				return err
 			}
 			defer f.Close()
-			return execOnContainer(containerName, f, command, args...)
+			return containerRuntime.ExecToFile(ctx, containerName, f, command, args...)
 		}
 	}
 	copyDirFn := func(containerDir, dirName string) func() error {
@@ -85,7 +90,8 @@ func (k DockerLogCollector) collectLogsFromNode(outputPath string, containerName
 
 			defer os.Remove(tempfileName)
 
-			err = execOnContainer(
+			err = containerRuntime.ExecToFile(
+				ctx,
 				containerName,
 				f,
 				"tar", "--hard-dereference", "--dereference", "--directory", containerDir, "--create", "--file", "-", ".",
@@ -140,34 +146,3 @@ func fileOnHost(path string) (*os.File, error) {
 	}
 	return os.Create(path)
 }
-
-// execOnContainer is an helper that runs a command on a CAPD node/container.
-func execOnContainer(containerName string, fileOnHost *os.File, command string, args ...string) error {
-	dockerArgs := []string{
-		"exec",
-		// run with privileges so we can remount etc..
-		// this might not make sense in the most general sense, but it is
-		// important to many kind commands
-		"--privileged",
-	}
-	// specify the container and command, after this everything will be
-	// args the the command in the container rather than to docker
-	dockerArgs = append(
-		dockerArgs,
-		containerName, // ... against the container
-		command,       // with the command specified
-	)
-	dockerArgs = append(
-		dockerArgs,
-		// finally, with the caller args
-		args...,
-	)
-
-	cmd := exec.Command("docker", dockerArgs...)
-	cmd.SetEnv("PATH", os.Getenv("PATH"))
-
-	cmd.SetStderr(fileOnHost)
-	cmd.SetStdout(fileOnHost)
-
-	return errors.WithStack(cmd.Run())
-}

test/framework/ginkgoextensions/output.go

@@ -23,6 +23,10 @@ import (
 	"github.com/onsi/ginkgo"
 )
 
+// TestOutput can be used for writing testing output.
+var TestOutput = ginkgo.GinkgoWriter
+
+// Byf provides formatted output to the GinkoWriter.
 func Byf(format string, a ...interface{}) {
 	ginkgo.By(fmt.Sprintf(format, a...))
 }

test/framework/kubetest/run.go

@@ -21,18 +21,20 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"os/exec"
 	"os/user"
 	"path"
 	"runtime"
 	"strconv"
 	"strings"
 
+	"github.com/onsi/ginkgo"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
 	"sigs.k8s.io/yaml"
 )
 
@@ -123,14 +125,6 @@ func Run(ctx context.Context, input RunInput) error {
 		return err
 	}
 
-	var testRepoListVolumeArgs []string
-	if input.KubeTestRepoListPath != "" {
-		testRepoListVolumeArgs, err = buildKubeTestRepoListArgs(kubetestConfigDir, input.KubeTestRepoListPath)
-		if err != nil {
-			return err
-		}
-	}
-
 	e2eVars := map[string]string{
 		"kubeconfig":           "/tmp/kubeconfig",
 		"provider":             "skeleton",
@@ -145,27 +139,55 @@ func Run(ctx context.Context, input RunInput) error {
 	if input.ConformanceImage == "" {
 		input.ConformanceImage = versionToConformanceImage(input.KubernetesVersion)
 	}
-	kubeConfigVolumeMount := volumeArg(tmpKubeConfigPath, "/tmp/kubeconfig")
-	outputVolumeMount := volumeArg(reportDir, "/output")
+	volumeMounts := map[string]string{
+		tmpKubeConfigPath: "/tmp/kubeconfig",
+		reportDir:         "/output",
+	}
 	user, err := user.Current()
 	if err != nil {
 		return errors.Wrap(err, "unable to determine current user")
 	}
-	userArg := user.Uid + ":" + user.Gid
-	entrypointArg := "--entrypoint=/usr/local/bin/ginkgo"
-	networkArg := "--network=kind"
-	e2eCmd := exec.Command("docker", "run", "--user", userArg, entrypointArg, kubeConfigVolumeMount, outputVolumeMount, "-t", networkArg)
-	if len(testRepoListVolumeArgs) > 0 {
-		e2eCmd.Args = append(e2eCmd.Args, testRepoListVolumeArgs...)
+	env := map[string]string{}
+
+	if input.KubeTestRepoListPath != "" {
+		tmpKubeTestRepoListPath := path.Join(kubetestConfigDir, "repo_list.yaml")
+		if err := copyFile(input.KubeTestRepoListPath, tmpKubeTestRepoListPath); err != nil {
+			return err
+		}
+		dest := "/tmp/repo_list.yaml"
+		env["KUBE_TEST_REPO_LIST"] = dest
+		volumeMounts[tmpKubeTestRepoListPath] = dest
+	}
+
+	// Formulate our command arguments
+	args := []string{}
+	args = append(args, ginkgoArgs...)
+	args = append(args, "/usr/local/bin/e2e.test")
+	args = append(args, "--")
+	args = append(args, e2eArgs...)
+	args = append(args, config.toFlags()...)
+
+	// Get our current working directory. Just for information, so we don't need
+	// to worry about errors at this point.
+	cwd, _ := os.Getwd()
+	ginkgoextensions.Byf("Running e2e test: dir=%s, command=%q", cwd, args)
+
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return errors.Wrap(err, "Unable to run conformance tests")
 	}
-	e2eCmd.Args = append(e2eCmd.Args, input.ConformanceImage)
-	e2eCmd.Args = append(e2eCmd.Args, ginkgoArgs...)
-	e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/e2e.test")
-	e2eCmd.Args = append(e2eCmd.Args, "--")
-	e2eCmd.Args = append(e2eCmd.Args, e2eArgs...)
-	e2eCmd.Args = append(e2eCmd.Args, config.toFlags()...)
-	e2eCmd = framework.CompleteCommand(e2eCmd, "Running e2e test", false)
-	if err := e2eCmd.Run(); err != nil {
+
+	err = containerRuntime.RunContainer(ctx, &container.RunContainerInput{
+		Image:           input.ConformanceImage,
+		Network:         "kind",
+		User:            user.Uid,
+		Group:           user.Gid,
+		Volumes:         volumeMounts,
+		EnvironmentVars: env,
+		CommandArgs:     args,
+		Entrypoint:      []string{"/usr/local/bin/ginkgo"},
+	}, ginkgo.GinkgoWriter)
+	if err != nil {
 		return errors.Wrap(err, "Unable to run conformance tests")
 	}
 	return framework.GatherJUnitReports(reportDir, input.ArtifactsDirectory)
@@ -235,27 +257,6 @@ func countClusterNodes(ctx context.Context, proxy framework.ClusterProxy) (int,
 	return len(nodeList.Items), nil
 }
 
-func isSELinuxEnforcing() bool {
-	dat, err := os.ReadFile("/sys/fs/selinux/enforce")
-	if err != nil {
-		return false
-	}
-	return string(dat) == "1"
-}
-
-func volumeArg(src, dest string) string {
-	volumeArg := "-v" + src + ":" + dest
-	if isSELinuxEnforcing() {
-		return volumeArg + ":z"
-	}
-	return volumeArg
-}
-
-func envArg(key, value string) string {
-	envArg := "-e" + key + "=" + value
-	return envArg
-}
-
 func versionToConformanceImage(kubernetesVersion string) string {
 	k8sVersion := strings.ReplaceAll(kubernetesVersion, "+", "_")
 	if isUsingCIArtifactsVersion(kubernetesVersion) {
@@ -274,16 +275,3 @@ func buildArgs(kv map[string]string, flagMarker string) []string {
 	}
 	return args
 }
-
-func buildKubeTestRepoListArgs(kubetestConfigDir, kubeTestRepoListPath string) ([]string, error) {
-	args := make([]string, 2)
-
-	tmpKubeTestRepoListPath := path.Join(kubetestConfigDir, "repo_list.yaml")
-	if err := copyFile(kubeTestRepoListPath, tmpKubeTestRepoListPath); err != nil {
-		return nil, err
-	}
-	dest := "/tmp/repo_list.yaml"
-	args[0] = envArg("KUBE_TEST_REPO_LIST", dest)
-	args[1] = volumeArg(tmpKubeTestRepoListPath, dest)
-	return args, nil
-}

test/framework/suite_helpers.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
 	. "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
@@ -87,8 +86,8 @@ func CreateJUnitReporterForProw(artifactsDirectory string) *reporters.JUnitRepor
 // CompleteCommand prints a command before running it. Acts as a helper function.
 // privateArgs when true will not print arguments.
 func CompleteCommand(cmd *exec.Cmd, desc string, privateArgs bool) *exec.Cmd {
-	cmd.Stderr = ginkgo.GinkgoWriter
-	cmd.Stdout = ginkgo.GinkgoWriter
+	cmd.Stderr = TestOutput
+	cmd.Stdout = TestOutput
 	if privateArgs {
 		Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd)
 	} else {

test/go.mod

@@ -5,8 +5,13 @@ go 1.16
 replace sigs.k8s.io/cluster-api => ../
 
 require (
+	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/containerd/containerd v1.5.2 // indirect
+	github.com/docker/docker v20.10.7+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/go-logr/logr v0.4.0
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.13.0
 	github.com/pkg/errors v0.9.1