diff --git a/hack/teste2e.sh b/hack/teste2e.sh new file mode 100644 index 00000000..09538030 --- /dev/null +++ b/hack/teste2e.sh @@ -0,0 +1,13 @@ +export KUBECONFIG=~/.kube/config +export KUBERNETES_SERVICE_HOST= +export KUBERNETES_SERVICE_PORT=6443 +export S3ENDPOINT= +export S3ACCESSKEY= +export S3SECRETKEY= +export ACK_GINKGO_DEPRECATIONS=1.16.5 +kubectl delete namespace radondb-mysql-e2e +kubectl get clusterrole|grep mysql|awk '{print "kubectl delete clusterrole "$1}'|bash +kubectl get clusterrolebindings|grep mysql|awk '{print "kubectl delete clusterrolebindings "$1}'|bash +kubectl get crd|grep mysql|awk '{print "kubectl delete crd "$1}'|bash + +make e2e-local \ No newline at end of file diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go new file mode 100644 index 00000000..bd9024d8 --- /dev/null +++ b/test/e2e/backup/backup.go @@ -0,0 +1,177 @@ +/* +Copyright 2021 RadonDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package backup + +import ( + "context" + "fmt" + "math/rand" + "regexp" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" + "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" + "github.com/radondb/radondb-mysql-kubernetes/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("MySQL Backup E2E Tests", func() { + f := framework.NewFramework("mcbackup-1") + _ = f + + var ( + cluster *apiv1alpha1.MysqlCluster + clusterKey types.NamespacedName + name string + backupSecret *corev1.Secret + //timeout time.Duration + POLLING = 2 * time.Second + backupDir string + leader string + follower string + ) + + BeforeEach(func() { + // be careful, mysql allowed hostname lenght is <63 + name = fmt.Sprintf("bk-%d", rand.Int31()/1000) + + //timeout = 350 * time.Second + + By("create a new backup secret") + backupSecret = f.NewBackupSecret() + Expect(f.Client.Create(context.TODO(), backupSecret)).To(Succeed(), "create backup secret failed") + By("creating a new cluster") + cluster = framework.NewCluster(name, f.Namespace.Name) + clusterKey = types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} + cluster.Spec.BackupSecretName = backupSecret.Name + Expect(f.Client.Create(context.TODO(), cluster)).To(Succeed(), + "failed to create cluster '%s'", cluster.Name) + By("waiting the cluster readiness") + framework.WaitClusterReadiness(f, cluster) + //get leader + for _, node := range cluster.Status.Nodes { + if node.RaftStatus.Role == string(utils.Leader) { + leader = strings.Split(node.Name, ".")[0] + } else if node.RaftStatus.Role == string(utils.Follower) { + follower = strings.Split(node.Name, ".")[0] + } + } + + Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), "failed to get cluster %s", cluster.Name) + + Eventually(f.RefreshClusterFn(cluster), f.Timeout, POLLING).Should( + framework.HaveClusterReplicas(2)) + Eventually(f.RefreshClusterFn(cluster), f.Timeout, POLLING).Should( + framework.HaveClusterCond(apiv1alpha1.ConditionReady, corev1.ConditionTrue)) + + // refresh cluster + Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), + "failed to get cluster %s", cluster.Name) + + }) + + It("backup to object store", func() { + //exectute sql command in mysql pod + By("executing insert data") + _, err := f.ExecSQLOnNode(*cluster, leader, "create table testtable (id int)") + Expect(err).To(BeNil()) + _, err = f.ExecSQLOnNode(*cluster, leader, "insert into testtable values (1),(2),(3)") + Expect(err).To(BeNil()) + rows, err := f.ExecSQLOnNode(*cluster, leader, "select * from testtable") + Expect(err).To(BeNil(), "failed to execute sql") + defer rows.Close() + var id int + ids := make([]int, 0) + for rows.Next() { + if err := rows.Scan(&id); err != nil { + Fail(err.Error()) + } + ids = append(ids, id) + } + Expect(ids).To(Equal([]int{1, 2, 3})) + Eventually(func() []int { + rows, _ := + f.ExecSQLOnNode(*cluster, follower, "select * from testtable ") + if rows == nil { + return nil + } + defer rows.Close() + var id int + ids := make([]int, 0) + for rows.Next() { + if err := rows.Scan(&id); err != nil { + Fail(err.Error()) + } + ids = append(ids, id) + } + return ids + }, f.Timeout, POLLING).Should(Equal([]int{1, 2, 3})) + By("executing a backup ") + // do the backup + backup := framework.NewBackup(cluster, leader) + Expect(f.Client.Create(context.TODO(), backup)).To(Succeed(), + "failed to create backup '%s'", backup.Name) + + Eventually(f.RefreshBackupFn(backup), f.Timeout, POLLING).Should( + framework.HaveBackupComplete()) + + if str, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, leader, "backup"); err == nil { + r, _ := regexp.Compile("backup_[0-9]+") + backupDir = r.FindString(str) + } + nameRestore := fmt.Sprintf("rs-%d", rand.Int31()/1000) + By("creating a new cluster from backup") + clusterRestore := framework.NewCluster(nameRestore, f.Namespace.Name) + clusterKeyRestore := types.NamespacedName{Name: clusterRestore.Name, Namespace: clusterRestore.Namespace} + clusterRestore.Spec.BackupSecretName = backupSecret.Name + clusterRestore.Spec.RestoreFrom = backupDir + Expect(f.Client.Create(context.TODO(), clusterRestore)).To(Succeed(), + "failed to create clusterRestore '%s'", clusterRestore.Name) + By("waiting the clusterRestore readiness") + framework.WaitClusterReadiness(f, clusterRestore) + Eventually(f.RefreshClusterFn(clusterRestore), f.Timeout, POLLING).Should( + framework.HaveClusterReplicas(2)) + Eventually(f.RefreshClusterFn(clusterRestore), f.Timeout, POLLING).Should( + framework.HaveClusterCond(apiv1alpha1.ConditionReady, corev1.ConditionTrue)) + Eventually(func() []int { + rows, _ := f.ExecSQLOnNode(*clusterRestore, fmt.Sprintf("%s-mysql-0", nameRestore), "select * from testtable ") + if rows == nil { + return nil + } + defer rows.Close() + var id int + ids := make([]int, 0) + for rows.Next() { + if err := rows.Scan(&id); err != nil { + Fail(err.Error()) + } + ids = append(ids, id) + } + return ids + }, f.Timeout, POLLING).Should(Equal([]int{1, 2, 3})) + + // refresh clusterRestore + Expect(f.Client.Get(context.TODO(), clusterKeyRestore, clusterRestore)).To(Succeed(), + "failed to get clusterRestore %s", clusterRestore.Name) + Expect(f.Client.Delete(context.TODO(), clusterRestore)).To(Succeed(), + "failed to delete clusterRestore '%s'", clusterRestore.Name) + }) + +}) diff --git a/test/e2e/cluster/cluster.go b/test/e2e/cluster/cluster.go new file mode 100644 index 00000000..5e9ba92b --- /dev/null +++ b/test/e2e/cluster/cluster.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 RadonDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "fmt" + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + + apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" + "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" +) + +const ( + POLLING = 2 * time.Second +) + +var _ = Describe("MySQL Cluster E2E Tests", func() { + f := framework.NewFramework("mc-1") + two := int32(2) + three := int32(3) + five := int32(5) + + var ( + cluster *apiv1alpha1.MysqlCluster + clusterKey types.NamespacedName + name string + ) + + BeforeEach(func() { + // Be careful, mysql allowed hostname lenght is <63. + name = fmt.Sprintf("cl-%d", rand.Int31()/1000) + + By("creating a new cluster") + cluster = framework.NewCluster(name, f.Namespace.Name) + clusterKey = types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} + Expect(f.Client.Create(context.TODO(), cluster)).To(Succeed(), "failed to create cluster '%s'", cluster.Name) + + By("testing the cluster readiness") + framework.WaitClusterReadiness(f, cluster) + Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), "failed to get cluster %s", cluster.Name) + }) + + It("scale out/in a cluster, 2 -> 3 -> 5 -> 3 -> 2", func() { + By("test cluster is ready after scale out 2 -> 3") + cluster.Spec.Replicas = &three + Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed()) + fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster)) + + By("test cluster is ready after scale out 3 -> 5") + cluster.Spec.Replicas = &five + Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed()) + fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster)) + + By("test cluster is ready after scale in 5 -> 3") + cluster.Spec.Replicas = &three + Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed()) + fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster)) + + By("test cluster is ready after scale in 3 -> 2") + cluster.Spec.Replicas = &two + Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed()) + fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster)) + }) + +}) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 26433a59..670d4803 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -17,9 +17,11 @@ limitations under the License. package e2e import ( + "context" "fmt" "os" "path" + "strings" "testing" "github.com/golang/glog" @@ -27,20 +29,49 @@ import ( "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeutils "k8s.io/apimachinery/pkg/util/runtime" + clientset "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper" ) const ( - operatorNamespace = "mysql-operator" + RadondbMysqlE2eNamespace = "radondb-mysql-e2e" ) +var releaseName = framework.RandStr(6) + var _ = SynchronizedBeforeSuite(func() []byte { - // BeforeSuite logic. + kubeCfg, err := framework.LoadConfig() + Expect(err).To(Succeed()) + // restClient := core.NewForConfigOrDie(kubeCfg).RESTClient() + + c, err := client.New(kubeCfg, client.Options{}) + if err != nil { + Fail(fmt.Sprintf("can't instantiate k8s client: %s", err)) + } + + // ginkgo node 1 + By("Install operator") + operatorNsObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: RadondbMysqlE2eNamespace, + }, + } + if err := c.Create(context.TODO(), operatorNsObj); err != nil { + if !strings.Contains(err.Error(), "already exists") { + Fail(fmt.Sprintf("can't create mysql-operator namespace: %s", err)) + } + } + framework.HelmInstallChart(releaseName, RadondbMysqlE2eNamespace) + return nil + }, func(data []byte) { // all other nodes framework.Logf("Running BeforeSuite actions on all node") @@ -50,7 +81,25 @@ var _ = SynchronizedBeforeSuite(func() []byte { // Here, the order of functions is reversed; first, the function which runs everywhere, // and then the function that only runs on the first Ginkgo node. var _ = SynchronizedAfterSuite(func() { - // AfterSuite logic. + // Run on all Ginkgo nodes + framework.Logf("Running AfterSuite actions on all node") + framework.RunCleanupActions() + + // get the kubernetes client + kubeCfg, err := framework.LoadConfig() + Expect(err).To(Succeed()) + + client, err := clientset.NewForConfig(kubeCfg) + Expect(err).NotTo(HaveOccurred()) + + By("Remove operator release") + framework.HelmPurgeRelease(releaseName, RadondbMysqlE2eNamespace) + + By("Delete operator namespace") + + if err := framework.DeleteNS(client, RadondbMysqlE2eNamespace, framework.DefaultNamespaceDeletionTimeout); err != nil { + framework.Failf(fmt.Sprintf("Can't delete namespace: %s", err)) + } }, func() { // Run only Ginkgo on node 1 framework.Logf("Running AfterSuite actions on node 1") @@ -84,13 +133,13 @@ func RunE2ETests(t *testing.T) { // add logs dumper if framework.TestContext.DumpLogsOnFailure { - rps = append(rps, NewLogsPodReporter(operatorNamespace, path.Join(framework.TestContext.ReportDir, + rps = append(rps, NewLogsPodReporter(RadondbMysqlE2eNamespace, path.Join(framework.TestContext.ReportDir, fmt.Sprintf("pods_logs_%d_%d.txt", config.GinkgoConfig.RandomSeed, config.GinkgoConfig.ParallelNode)))) } } else { // if reportDir is not specified then print logs to stdout if framework.TestContext.DumpLogsOnFailure { - rps = append(rps, NewLogsPodReporter(operatorNamespace, "")) + rps = append(rps, NewLogsPodReporter(RadondbMysqlE2eNamespace, "")) } } return diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 36b846a2..39c81f42 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -19,8 +19,10 @@ package e2e import ( "testing" + // _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/cluster" + _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/backup" "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" - _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase" + // _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase" ) func init() { diff --git a/test/e2e/framework/backup_util.go b/test/e2e/framework/backup_util.go new file mode 100644 index 00000000..fef3160a --- /dev/null +++ b/test/e2e/framework/backup_util.go @@ -0,0 +1,108 @@ +/* +Copyright 2018 Pressinfra SRL + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package framework + +import ( + "context" + "fmt" + "os" + + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + apiv1alpha "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func GetS3EndPointName() string { + S3 := os.Getenv("S3ENDPOINT") + if len(S3) == 0 { + Logf("S3ENDPOINT not set! Backups tests will not work") + } + return S3 +} + +func GetS3AccessKey() string { + S3AccessKey := os.Getenv("S3ACCESSKEY") + if len(S3AccessKey) == 0 { + Logf("S3ACCESSKEY not set! Backups tests will not work") + } + return S3AccessKey +} + +func GetS3SecretKey() string { + S3SecretKey := os.Getenv("S3SECRETKEY") + if len(S3SecretKey) == 0 { + Logf("S3SECRETKEY not set! Backups tests will not work") + } + return S3SecretKey +} + +func (f *Framework) NewBackupSecret() *corev1.Secret { + // s3-endpoint: + // s3-access-key: + // s3-secret-key: + // s3-bucket: + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-backup-secret", f.BaseName), + Namespace: f.Namespace.Name, + }, + StringData: map[string]string{ + "s3-endpoint": GetS3EndPointName(), + "s3-access-key": GetS3AccessKey(), + "s3-secret-key": GetS3SecretKey(), + "s3-bucket": "radondb-backups", + }, + Type: corev1.SecretTypeOpaque, + } +} + +func NewBackup(cluster *apiv1alpha.MysqlCluster, hostname string) *apiv1alpha.Backup { + return &apiv1alpha.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Name, + Namespace: cluster.Namespace, + }, + Spec: apiv1alpha.BackupSpec{ + ClusterName: cluster.Name, + HostName: hostname, + Image: TestContext.SidecarImage, + }, + } +} + +func (f *Framework) RefreshBackupFn(backup *apiv1alpha.Backup) func() *apiv1alpha.Backup { + return func() *apiv1alpha.Backup { + key := types.NamespacedName{ + Name: backup.Name, + Namespace: backup.Namespace, + } + b := &apiv1alpha.Backup{} + f.Client.Get(context.TODO(), key, b) + return b + } +} + +func HaveBackupComplete() gomegatypes.GomegaMatcher { + return PointTo(MatchFields(IgnoreExtras, Fields{ + "Status": MatchFields(IgnoreExtras, Fields{ + "Completed": Equal(true), + })}, + )) +} diff --git a/test/e2e/framework/cluster_util.go b/test/e2e/framework/cluster_util.go new file mode 100644 index 00000000..8628fcd7 --- /dev/null +++ b/test/e2e/framework/cluster_util.go @@ -0,0 +1,271 @@ +/* +Copyright 2018 Pressinfra SRL + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "database/sql" + "fmt" + "time" + + _ "github.com/go-sql-driver/mysql" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + k8score "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" + pf "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/portforward" + "github.com/radondb/radondb-mysql-kubernetes/utils" +) + +var ( + POLLING = 2 * time.Second + TIMEOUT = time.Minute + FAILOVERPOLLING = 200 * time.Millisecond + FAILOVERTIMEOUT = 2 * time.Minute +) + +func (f *Framework) ClusterEventuallyCondition(cluster *apiv1alpha1.MysqlCluster, + condType apiv1alpha1.ClusterConditionType, status corev1.ConditionStatus, timeout time.Duration) { + Eventually(func() []apiv1alpha1.ClusterCondition { + key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} + if err := f.Client.Get(context.TODO(), key, cluster); err != nil { + return nil + } + return cluster.Status.Conditions + }, timeout, POLLING).Should(ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(condType), + "Status": Equal(status), + })), "Testing cluster '%s' for condition %s to be %s", cluster.Name, condType, status) + +} + +func (f *Framework) NodeEventuallyCondition(cluster *apiv1alpha1.MysqlCluster, nodeName string, + condType apiv1alpha1.NodeConditionType, status corev1.ConditionStatus, timeout time.Duration) { + Eventually(func() []apiv1alpha1.NodeCondition { + key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} + if err := f.Client.Get(context.TODO(), key, cluster); err != nil { + return nil + } + + for _, ns := range cluster.Status.Nodes { + if ns.Name == nodeName { + return ns.Conditions + } + } + + return nil + }, timeout, POLLING).Should(ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(condType), + "Status": Equal(status), + })), "Testing node '%s' of the cluster '%s'", cluster.Name, nodeName) +} + +func (f *Framework) ClusterEventuallyReplicas(cluster *apiv1alpha1.MysqlCluster, timeout time.Duration) { + Eventually(func() int { + cl := &apiv1alpha1.MysqlCluster{} + f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl) + return cl.Status.ReadyNodes + }, timeout, POLLING).Should(Equal(int(*cluster.Spec.Replicas)), "Not ready replicas of cluster '%s'", cluster.Name) +} + +func (f *Framework) ClusterEventuallyRaftStatus(cluster *apiv1alpha1.MysqlCluster) { + Eventually(func() bool { + cl := &apiv1alpha1.MysqlCluster{} + f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl) + return isXenonReadiness(cl) + }, TIMEOUT, POLLING).Should(BeTrue(), "Not ready xenon of cluster '%s'", cluster.Name) +} + +// isXenonReadiness determine whether the role of the cluster is normal. +// 1. Cluster must have Leader node. +// 2. Cluster can only have a Leader node. +// 3. There are only two roles of Leader and Follower in the cluster. +func isXenonReadiness(cluster *apiv1alpha1.MysqlCluster) bool { + leader := []string{} + follower := []string{} + for _, node := range cluster.Status.Nodes { + if node.RaftStatus.Role == string(utils.Leader) { + leader = append(leader, node.Name) + } else if node.RaftStatus.Role == string(utils.Follower) { + follower = append(follower, node.Name) + } else { + return false + } + } + if len(leader) != 1 { + return false + } + if len(follower) != len(cluster.Status.Nodes)-len(leader) { + return false + } + return true +} + +// HaveClusterCond is a helper func that returns a matcher to check for an existing condition in a ClusterCondition list. +func HaveClusterCond(condType apiv1alpha1.ClusterConditionType, status corev1.ConditionStatus) gomegatypes.GomegaMatcher { + return PointTo(MatchFields(IgnoreExtras, Fields{ + "Status": MatchFields(IgnoreExtras, Fields{ + "Conditions": ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(condType), + "Status": Equal(status), + })), + })}, + )) +} + +func (f *Framework) RefreshClusterFn(cluster *apiv1alpha1.MysqlCluster) func() *apiv1alpha1.MysqlCluster { + return func() *apiv1alpha1.MysqlCluster { + key := types.NamespacedName{ + Name: cluster.Name, + Namespace: cluster.Namespace, + } + c := &apiv1alpha1.MysqlCluster{} + f.Client.Get(context.TODO(), key, c) + return c + } +} + +// HaveClusterRepliacs matcher for replicas +func HaveClusterReplicas(replicas int) gomegatypes.GomegaMatcher { + return PointTo(MatchFields(IgnoreExtras, Fields{ + "Status": MatchFields(IgnoreExtras, Fields{ + "ReadyNodes": Equal(replicas), + }), + })) +} + +// GetClusterLabels returns labels.Set for the given cluster +func GetClusterLabels(cluster *apiv1alpha1.MysqlCluster) labels.Set { + labels := labels.Set{ + "mysql.radondb.com/cluster": cluster.Name, + "app.kubernetes.io/name": "mysql", + } + + return labels +} + +func (f *Framework) GetClusterPVCsFn(cluster *apiv1alpha1.MysqlCluster) func() []corev1.PersistentVolumeClaim { + return func() []corev1.PersistentVolumeClaim { + pvcList := &corev1.PersistentVolumeClaimList{} + lo := &client.ListOptions{ + Namespace: cluster.Namespace, + LabelSelector: labels.SelectorFromSet(GetClusterLabels(cluster)), + } + f.Client.List(context.TODO(), pvcList, lo) + return pvcList.Items + } +} + +func NewCluster(name, ns string) *apiv1alpha1.MysqlCluster { + two := int32(2) + return &apiv1alpha1.MysqlCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: apiv1alpha1.MysqlClusterSpec{ + Replicas: &two, + PodPolicy: apiv1alpha1.PodPolicy{ + SidecarImage: TestContext.SidecarImage, + ImagePullPolicy: "Always", + }, + }, + } +} + +func (f *Framework) ExecSQLOnNode(cluster apiv1alpha1.MysqlCluster, podName, query string) (*sql.Rows, error) { + kubeCfg, err := LoadConfig() + Expect(err).NotTo(HaveOccurred()) + + user := cluster.Spec.MysqlOpts.User + password := cluster.Spec.MysqlOpts.Password + + client := k8score.NewForConfigOrDie(kubeCfg).RESTClient() + tunnel := pf.NewTunnel(client, kubeCfg, cluster.Namespace, + podName, + 3306, + ) + defer tunnel.Close() + + err = tunnel.ForwardPort() + Expect(err).NotTo(HaveOccurred(), "Failed setting up port-forarding for pod: %s", podName) + + dsn := fmt.Sprintf("%s:%s@tcp(localhost:%d)/radondb?timeout=20s&multiStatements=true", user, password, tunnel.Local) + db, err := sql.Open("mysql", dsn) + Expect(err).To(Succeed(), "Failed connection to mysql DSN: %s", dsn) + defer db.Close() + + rows, err := db.Query(query) + if err != nil { + return nil, fmt.Errorf("err: %s, query: %s", err, query) + } + return rows, nil +} + +func (f Framework) IsPodExist(roleLabel map[string]string, cluster *apiv1alpha1.MysqlCluster) bool { + lo := &client.ListOptions{ + Namespace: cluster.Namespace, + LabelSelector: labels.SelectorFromSet(GetClusterLabels(cluster)), + } + roleRequirement, err := labels.NewRequirement("role", selection.Equals, []string{roleLabel["role"]}) + if err != nil { + fmt.Sprintln("failed to create roleRequirement") + return false + } + lo.LabelSelector.Add(*roleRequirement) + podList, err := f.ClientSet.CoreV1().Pods(cluster.Namespace).List(context.TODO(), *lo.AsListOptions()) + if err != nil { + fmt.Sprintln("failed to get pod") + return false + } + if len(podList.Items) > 0 { + return true + } + return false +} + +func (f Framework) WaitServiceAvailable(clusterKey types.NamespacedName, roleLabel map[string]string) { + Eventually(func() bool { + cluster := apiv1alpha1.MysqlCluster{} + f.Client.Get(context.TODO(), clusterKey, &cluster) + return f.IsPodExist(roleLabel, &cluster) + }, FAILOVERTIMEOUT, FAILOVERPOLLING).Should(BeTrue(), "service is unavailable") +} + +// WaitClusterReadiness determine whether the cluster is ready. +func WaitClusterReadiness(f *Framework, cluster *apiv1alpha1.MysqlCluster) time.Duration { + startTime := time.Now() + timeout := f.Timeout + if *cluster.Spec.Replicas > 0 { + timeout = time.Duration(*cluster.Spec.Replicas) * f.Timeout + } + // Wait for pods to be ready. + f.ClusterEventuallyReplicas(cluster, timeout) + // Wait for xenon to be ready. + f.ClusterEventuallyRaftStatus(cluster) + // Wait for condition to be ready. + f.ClusterEventuallyCondition(cluster, apiv1alpha1.ConditionReady, corev1.ConditionTrue, f.Timeout) + return time.Since(startTime) +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index e864d15e..ba785b3b 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,6 +35,7 @@ import ( const ( maxKubectlExecRetries = 5 DefaultNamespaceDeletionTimeout = 10 * time.Minute + RadondbMysqlE2eNamespace = "radondb-mysql-e2e" ) type Framework struct { @@ -85,13 +87,11 @@ func (f *Framework) BeforeEach() { Expect(err).NotTo(HaveOccurred()) if !f.SkipNamespaceCreation { - namespace, err := f.CreateNamespace(map[string]string{ - "e2e-framework": f.BaseName, - }) - Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("create a namespace api object (%s)", namespace.Name)) - - f.Namespace = namespace + f.Namespace = &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: RadondbMysqlE2eNamespace, + }, + } } } diff --git a/test/e2e/framework/portforward/portforward.go b/test/e2e/framework/portforward/portforward.go new file mode 100644 index 00000000..d07735e5 --- /dev/null +++ b/test/e2e/framework/portforward/portforward.go @@ -0,0 +1,116 @@ +/* +Copyright 2018 Pressinfra SRL + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package portforward + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +type Tunnel struct { + Local int + Remote int + Namespace string + PodName string + Out io.Writer + stopChan chan struct{} + readyChan chan struct{} + config *rest.Config + client rest.Interface +} + +func NewTunnel(client rest.Interface, config *rest.Config, namespace, podName string, remote int) *Tunnel { + return &Tunnel{ + config: config, + client: client, + Namespace: namespace, + PodName: podName, + Remote: remote, + stopChan: make(chan struct{}, 1), + readyChan: make(chan struct{}, 1), + Out: ioutil.Discard, + } +} + +func (t *Tunnel) ForwardPort() error { + u := t.client.Post(). + Resource("pods"). + Namespace(t.Namespace). + Name(t.PodName). + SubResource("portforward").URL() + + transport, upgrader, err := spdy.RoundTripperFor(t.config) + if err != nil { + return err + } + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", u) + + local, err := getAvailablePort() + if err != nil { + return fmt.Errorf("could not find an available port: %s", err) + } + t.Local = local + + ports := []string{fmt.Sprintf("%d:%d", t.Local, t.Remote)} + + pf, err := portforward.New(dialer, ports, t.stopChan, t.readyChan, t.Out, t.Out) + if err != nil { + return err + } + + errChan := make(chan error) + go func() { + errChan <- pf.ForwardPorts() + }() + + select { + case err = <-errChan: + return fmt.Errorf("forwarding ports: %v", err) + case <-pf.Ready: + return nil + } +} + +func (t *Tunnel) Close() { + close(t.stopChan) +} + +func getAvailablePort() (int, error) { + l, err := net.Listen("tcp", ":0") + if err != nil { + return 0, err + } + defer l.Close() + + _, p, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return 0, err + } + port, err := strconv.Atoi(p) + if err != nil { + return 0, err + } + return port, err +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 511ba4b6..cd1424c4 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -58,7 +58,7 @@ func RegisterCommonFlags() { flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar) flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kuberentes. If unset, will use value from 'current-context'") - flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Optional directory to store junit and pod logs output in. If not specified, no junit or logs files will be output") + flag.StringVar(&TestContext.ReportDir, "report-dir", "logs", "Optional directory to store junit and pod logs output in. If not specified, no junit or logs files will be output") flag.StringVar(&TestContext.ChartPath, "operator-chart-path", "../../charts/mysql-operator", "The chart name or path for mysql operator") flag.StringVar(&TestContext.OperatorImageTag, "operator-image-tag", "latest", "Image tag for mysql operator.") diff --git a/test/e2e/reporter.go b/test/e2e/reporter.go index bad5e5eb..676e819e 100644 --- a/test/e2e/reporter.go +++ b/test/e2e/reporter.go @@ -44,10 +44,6 @@ type podLogReporter struct { out io.Writer } -var radondbMysqlTestLabel = map[string]string{ - "app.kubernetes.io/managed-by": "mysql.radondb.com", -} - // NewLogsPodReporter writes the logs for all pods in the specified namespace. // if path is specified then the logs are written to that path, else logs are // written to GinkgoWriter @@ -101,7 +97,7 @@ func (r *podLogReporter) SpecDidComplete(specSummary *types.SpecSummary) { fmt.Fprintf(r.out, "## Start test: %v\n", specSummary.ComponentTexts) - LogPodsWithLabels(client, r.namespace, radondbMysqlTestLabel, specSummary.RunTime, r.out) + LogPodsWithLabels(client, r.namespace, nil, specSummary.RunTime, r.out) fmt.Fprintf(r.out, "## END test\n")