*: support juicefs #745

Author: acekingke

Dockerfile.sidecar

@@ -43,13 +43,20 @@ RUN set -ex; \
 ARG XTRABACKUP_PKG=percona-xtrabackup-24
 RUN set -ex; \
     apt-get update; \
-    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
+    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server sqlite3; \
     wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
     dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
     apt-get update; \
     apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
+#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
+# COPY juicefs/juicefs /usr/local/bin/juicefs
+RUN wget --no-check-certificate "https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz" && tar -zxf "juicefs-1.0.2-linux-amd64.tar.gz" ;\
+    mv juicefs /usr/local/bin/juicefs; \
+    chmod +x /usr/local/bin/juicefs ; mkdir -p /run/sshd; \
+    mkdir -p /root/.ssh; \
+    chmod 700 /root/.ssh 
 WORKDIR /
 COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
-ENTRYPOINT ["sidecar"]
+COPY script/*.sh /
+CMD [ "sidecar" ]

api/v1alpha1/backup_types.go

@@ -21,6 +21,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type JuiceOpt struct {
+	// sqlite or redis
+	JuiceMeta string `json:"juiceMeta"`
+	// backupSecrete name for S3
+	BackupSecretName string `json:"backupSecretName"`
+	JuiceName        string `json:"juiceName"`
+}
+
 // This is the backup Job CRD.
 // BackupSpec defines the desired state of Backup
 type BackupSpec struct {
@@ -40,6 +48,9 @@ type BackupSpec struct {
 	// +optional
 	NFSServerAddress string `json:"nfsServerAddress,omitempty"`
 
+	// Represents the juicefs parameters which need.
+	// +optional
+	JuiceOpt *JuiceOpt `json:"juiceOpt,omitempty"`
 	// ClusterName represents the cluster name to backup
 	ClusterName string `json:"clusterName"`
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -17,13 +17,16 @@ limitations under the License.
 package syncer
 
 import (
+	"context"
 	"fmt"
+	"strings"
 
 	"github.com/presslabs/controller-util/pkg/syncer"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	v1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
@@ -33,6 +36,7 @@ import (
 )
 
 type jobSyncer struct {
+	client client.Client
 	job    *batchv1.Job
 	backup *backup.Backup
 }
@@ -50,6 +54,7 @@ func NewJobSyncer(c client.Client, backup *backup.Backup) syncer.Interface {
 	}
 
 	sync := &jobSyncer{
+		client: c,
 		job:    obj,
 		backup: backup,
 	}
@@ -174,6 +179,10 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
 				MountPath: utils.XtrabckupLocal,
 			},
 		}
+	} else if s.backup.Spec.JuiceOpt != nil {
+		// Deal it for juiceOpt
+		s.buildJuicefsBackPod(&in)
+
 	} else {
 		// in.Containers[0].ImagePullPolicy = s.opt.ImagePullPolicy
 		in.Containers[0].Args = []string{
@@ -238,3 +247,84 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
 	}
 	return in
 }
+
+func (s *jobSyncer) buildJuicefsBackPod(in *corev1.PodSpec) error {
+	// add volumn about pvc
+	var defMode int32 = 0600
+	var err error
+	var cmdstr string
+	in.Volumes = []corev1.Volume{
+		{
+			Name: utils.SShVolumnName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName:  fmt.Sprintf("%s-ssh-key", s.backup.Spec.ClusterName),
+					DefaultMode: &defMode,
+				},
+			},
+		},
+	}
+
+	in.Containers[0].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      utils.SShVolumnName,
+			MountPath: utils.SshVolumnPath,
+		},
+	}
+
+	// PodName.clusterName-mysql.Namespace
+	// sample-mysql-0.sample-mysql.default
+	hostname := fmt.Sprintf("%s.%s-mysql.%s", s.backup.Spec.HostName, s.backup.Spec.ClusterName, s.backup.Namespace)
+	if cmdstr, err = s.buildJuicefsCmd(s.backup.Spec.JuiceOpt.BackupSecretName); err != nil {
+		return err
+	}
+
+	in.Containers[0].Command = []string{"bash", "-c", "--", `cp  /etc/secret-ssh/* /root/.ssh
+chmod 600 /root/.ssh/authorized_keys ;` +
+		strings.Join([]string{
+			"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", hostname, cmdstr,
+		}, " ")}
+
+	return nil
+}
+
+func (s *jobSyncer) buildJuicefsCmd(secName string) (string, error) {
+	juiceopt := s.backup.Spec.JuiceOpt
+	secret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secName,
+			Namespace: s.backup.Namespace,
+		},
+	}
+	err := s.client.Get(context.TODO(),
+		types.NamespacedName{Namespace: s.backup.Namespace,
+			Name: secName}, secret)
+
+	if err != nil {
+		return "", err
+	}
+	url, bucket := secret.Data["s3-endpoint"], secret.Data["s3-bucket"]
+	accesskey, secretkey := secret.Data["s3-access-key"], secret.Data["s3-secret-key"]
+	juicebucket := utils.InstallBucket(string(url), string(bucket))
+	cmdstr := fmt.Sprintf(`<<EOF
+	export CLUSTER_NAME=%s
+	juicefs format --storage s3 \
+    --bucket  %s \
+    --access-key %s \
+    --secret-key %s \
+    %s \
+    %s`, s.backup.Spec.ClusterName, juicebucket, accesskey, secretkey, juiceopt.JuiceMeta, juiceopt.JuiceName)
+	cmdstr += fmt.Sprintf(`
+	juicefs mount -d %s /%s/
+	`, juiceopt.JuiceMeta, juiceopt.JuiceName)
+	cmdstr += fmt.Sprintf(`
+	source /backup.sh
+    backup
+	juicefs umount /%s/
+EOF`, juiceopt.JuiceName)
+	return cmdstr, nil
+}

backup/syncer/job.go

@@ -70,6 +70,22 @@ spec:
                 default: radondb/mysql57-sidecar:v2.3.0
                 description: To specify the image that will be used for sidecar container.
                 type: string
+              juiceOpt:
+                description: Represents the juicefs parameters which need.
+                properties:
+                  backupSecretName:
+                    description: backupSecrete name for S3
+                    type: string
+                  juiceMeta:
+                    description: sqlite or redis
+                    type: string
+                  juiceName:
+                    type: string
+                required:
+                - backupSecretName
+                - juiceMeta
+                - juiceName
+                type: object
               nfsServerAddress:
                 description: Represents the ip address of the nfs server.
                 type: string

charts/mysql-operator/crds/mysql.radondb.com_backups.yaml

@@ -70,6 +70,22 @@ spec:
                 default: radondb/mysql57-sidecar:v2.3.0
                 description: To specify the image that will be used for sidecar container.
                 type: string
+              juiceOpt:
+                description: Represents the juicefs parameters which need.
+                properties:
+                  backupSecretName:
+                    description: backupSecrete name for S3
+                    type: string
+                  juiceMeta:
+                    description: sqlite or redis
+                    type: string
+                  juiceName:
+                    type: string
+                required:
+                - backupSecretName
+                - juiceMeta
+                - juiceName
+                type: object
               nfsServerAddress:
                 description: Represents the ip address of the nfs server.
                 type: string

config/crd/bases/mysql.radondb.com_backups.yaml

@@ -24,7 +24,9 @@ import (
 	"github.com/presslabs/controller-util/pkg/syncer"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	policyv1beta1 "k8s.io/api/policy/v1beta1"
+
+	// policyv1beta1 "k8s.io/api/policy/v1beta1"
+	policyv1beta1 "k8s.io/api/policy/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/labels"
@@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
+	secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
+	if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// Todo: modify mysql cm will trigger rolling update but it will not be applied.
 	cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
 	sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion

controllers/mysqlcluster_controller.go

@@ -0,0 +1,98 @@
+English | [简体中文](../zh-cn/juicefs_backup_and_restore.md) | 
+
+目录
+=============
+ * [juiceopt backup]()
+    * [prerequest](#startup-juiceopt-backup)
+    * [configuration](#fill-backup-crds-yaml-configs)
+    * [running backup](#running-backup)
+ 
+* [restore](#restore)
+  * [prepare job](#prerequest-for-restore)
+  * [add configmap](#and-config-map)
+  * [config mysql cluster](#config-mysql-clusters-yaml)
+  * [app the cluster restore from backup](#use-kubectl-apply-the-yaml)
+
+# startup juiceopt backup
+## prerequire
+ 1. prepare S3 strorage （if you want other types of storage,  reference the juicfs document），Obtain the access-key and secret-key . In the example of this article, it use minio ,and the instance is minio-1668754867, and the bucket named `test` so the url is http://test.minio-1668754867.minio:9000/ , you can modify it according your situations, and how to do you can refer to juicefs documents.
+    
+ 2. Install the redis , although juicefs also support sqlite as meta data storage, buf if you do so, you should sync the meta file from s3 at first,and I do not recommend it. redis url is the form as follow：
+ ```
+ redis://<redis-server-name/IP>:<port>/<NO of database>
+ ```
+in the example of this article, redis-server-name is  redis-leader, the number of database is 1, So the redis url is  `redis://redis-leader:6379/1`
+
+ 3. Verfiy whether it works: suppose the backup directory is juicefs , you can login in  Pod's backup container , execute commanas as follow：
+  
+  ```
+  juicefs format --storage s3 \
+    --bucket http://test.minio-1668754867.minio:9000/  \
+    --access-key <your access key> \
+    --secret-key <your secrete key> > \
+    redis://redis-leader:6379/1 \
+    juicefs
+  ```
+ then execute ：
+ `juicefs mount -d redis://redis-leader:6379/1 /juicefs` 
+
+check whether juicefs is exist, write files, and check S3 storage whether has changed.
+
+## fill backup crd's yaml configs
+In backup crd's yaml file, such as in samples/mysql_v1alpha_backup.yaml, add fields information under spec:
+
+```
+  juiceOpt:
+    juiceMeta: <fill your redis url>
+    backupSecretName: <S3's secret name, for example:sample-backup-secret>
+    juiceName: <juicefs backup directory>
+```
+for example:
+```
+  juiceOpt:
+    juiceMeta: "redis://redis-leader:6379/1"
+    backupSecretName: sample-backup-secret
+    juiceName: juicefs
+```
+Others refer to [backup and restore config](./backup_and_restoration_s3.md)
+
+## Running backup.
+
+use command `kubectl apply -f <your backup crd's yaml>` , for examples：
+
+```
+ kubectl apply -f config/samples/mysql_v1alpha1_backup.yaml 
+``` 
+
+# Restore
+## prerequest for restore
+  I suppose that the cluster you want restore is `sample2`
+###  and `config map`
+   1. At first give the `config map` a name,name's form is <name of restore cluster>-restore, this article suppose that cluster name is sample2, so `config map`'s name is  `sample2-restore`
+   2. Create config map
+    * prepare for juiceopt parameters: 
+        build a yaml file, named `juiceopt.yaml`, fill it with:
+        ```
+        juiceMeta: <redis url>
+        backupSecretName: <S3's secret name>
+        juiceName: <backup directory under S3 bucket>
+        ```
+        for example,  in the example of this article, juiceopt.yaml is:
+        ```
+        juiceMeta: "redis://redis-leader:6379/1"
+        backupSecretName: sample-backup-secret
+        juiceName: juicefs
+        ```
+    * use `kubectl create configmap` create a configmap
+        configmap has two keys , `from` and `juice.opt`  that respectively indicate the cluster has been backuped which we should restore from, and the juice parameter.
+        but `date` key is optional, it indicates the time where restore to (format is:"2006-01-02 09:07:41"), if it does not have got this key , it will restore to now, use the commands as follows:
+        `kubectl create configmap sample2-restore --from-literal=from=sample --from-file="juice.opt"=./juiceopt.yaml `
+
+
+###  config mysql cluster's yaml
+     in the example of this article, we suppose the cluster need to restore is sample2, the config method can refer to [radondb cluster configuration](./deploy_radondb-mysql_operator_on_k8s.md)
+###  use kubectl apply the yaml
+    use `kubectl apply ` apply the yaml file, for the example, use the commands as follow:
+
+    `kubectl apply -f config/samples/mysql_v1alpha1_mysqlcluster.yaml ` 
+