Skip to content

Commit 52b3ee8

Browse files
committed
*: support juicefs radondb#745
1 parent 41b6f3c commit 52b3ee8

12 files changed

+264
-9
lines changed

Dockerfile.sidecar

+9-3
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,19 @@ RUN set -ex; \
4343
ARG XTRABACKUP_PKG=percona-xtrabackup-24
4444
RUN set -ex; \
4545
apt-get update; \
46-
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
46+
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \
4747
wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
4848
dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
4949
apt-get update; \
5050
apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
5151
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
52-
52+
#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
53+
COPY juicefs/juicefs /usr/local/bin/juicefs
54+
RUN chmod +x /usr/local/bin/juicefs && mkdir -p /run/sshd; \
55+
mkdir -p /root/.ssh; \
56+
chmod 700 /root/.ssh
5357
WORKDIR /
5458
COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
55-
ENTRYPOINT ["sidecar"]
59+
COPY script/sshd.sh /sshd.sh
60+
COPY script/backup.sh /backup.sh
61+
CMD [ "sidecar" ]

controllers/mysqlcluster_controller.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ import (
2424
"github.com/presslabs/controller-util/pkg/syncer"
2525
appsv1 "k8s.io/api/apps/v1"
2626
corev1 "k8s.io/api/core/v1"
27-
policyv1beta1 "k8s.io/api/policy/v1beta1"
27+
28+
// policyv1beta1 "k8s.io/api/policy/v1beta1"
29+
policyv1beta1 "k8s.io/api/policy/v1"
2830
rbacv1 "k8s.io/api/rbac/v1"
2931
"k8s.io/apimachinery/pkg/api/errors"
3032
"k8s.io/apimachinery/pkg/labels"
@@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
125127
return ctrl.Result{}, err
126128
}
127129

130+
secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
131+
if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
132+
return ctrl.Result{}, err
133+
}
134+
128135
// Todo: modify mysql cm will trigger rolling update but it will not be applied.
129136
cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
130137
sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion

mysqlcluster/container/backup.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func (c *backupSidecar) getImage() string {
3838
}
3939

4040
func (c *backupSidecar) getCommand() []string {
41-
return []string{"sidecar", "http"}
41+
return []string{"sh", "-c", "/sshd.sh ; sidecar http"}
4242
}
4343

4444
func (c *backupSidecar) getEnvVars() []corev1.EnvVar {
@@ -147,5 +147,13 @@ func (c *backupSidecar) getVolumeMounts() []corev1.VolumeMount {
147147
Name: utils.SysLocalTimeZone,
148148
MountPath: utils.SysLocalTimeZoneMountPath,
149149
},
150+
{
151+
Name: utils.SysFuseVolume,
152+
MountPath: utils.SysFuseVolumnMountPath,
153+
},
154+
{
155+
Name: utils.SShVolumnName,
156+
MountPath: utils.SshVolumnPath,
157+
},
150158
}
151159
}

mysqlcluster/container/container.go

+10
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ func getStartupProbe(name string) *corev1.Probe {
6363
// EnsureContainer ensure a container by the giving name.
6464
func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container {
6565
var ctr container
66+
var security *corev1.SecurityContext = nil
6667
switch name {
6768
case utils.ContainerInitSidecarName:
6869
ctr = &initSidecar{c, name}
@@ -80,6 +81,14 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
8081
ctr = &auditLog{c, name}
8182
case utils.ContainerBackupName:
8283
ctr = &backupSidecar{c, name}
84+
needAdmin := true
85+
security = &corev1.SecurityContext{
86+
Privileged: &needAdmin,
87+
Capabilities: &corev1.Capabilities{
88+
Add: []corev1.Capability{"CAP_SYS_ADMIN",
89+
"DAC_READ_SEARCH",
90+
},
91+
}}
8392
}
8493

8594
return corev1.Container{
@@ -95,5 +104,6 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
95104
ReadinessProbe: ctr.getReadinessProbe(),
96105
StartupProbe: getStartupProbe(name),
97106
VolumeMounts: ctr.getVolumeMounts(),
107+
SecurityContext: security,
98108
}
99109
}

mysqlcluster/genkey.sh

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
2+
# ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no sample-mysql-0.sample-mysql.default
3+
kubectl create secret generic sample-ssh-key --from-file=id_ecdsa=/root/.ssh/id_ecdsa --from-file=authorized_keys=/root/.ssh/id_ecdsa.pub
4+
5+
/usr/sbin/sshd -D -e -f /etc/ssh/sshd_config
6+
mkdir -p /root/.ssh
7+
chmod 700 /root/.ssh
8+
cp /etc/secret-ssh/* /root/.ssh
9+
chmod 600 /root/.ssh/authorized_keys
10+
11+
12+
cat <<EOF >>sshd_config
13+
PermitRootLogin yes
14+
PasswordAuthentication no
15+
PermitEmptyPasswords no
16+
ChallengeResponseAuthentication no
17+
GSSAPIAuthentication no
18+
UseDNS no
19+
UsePAM yes
20+
EOF

mysqlcluster/mysqlcluster.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
180180
},
181181
)
182182
}
183-
183+
var defMode int32 = 0600
184184
volumes = append(volumes,
185185
corev1.Volume{
186186
Name: utils.MysqlConfVolumeName,
@@ -246,6 +246,23 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
246246
},
247247
},
248248
},
249+
corev1.Volume{
250+
Name: utils.SysFuseVolume,
251+
VolumeSource: corev1.VolumeSource{
252+
HostPath: &corev1.HostPathVolumeSource{
253+
Path: "/dev/fuse",
254+
},
255+
},
256+
},
257+
corev1.Volume{
258+
Name: utils.SShVolumnName,
259+
VolumeSource: corev1.VolumeSource{
260+
Secret: &corev1.SecretVolumeSource{
261+
SecretName: c.GetNameForResource(utils.SShKey),
262+
DefaultMode: &defMode,
263+
},
264+
},
265+
},
249266
)
250267
// add the nfs volumn mount
251268
if len(c.Spec.NFSServerAddress) != 0 {
@@ -328,6 +345,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
328345
return fmt.Sprintf("%s-metrics", c.Name)
329346
case utils.Secret:
330347
return fmt.Sprintf("%s-secret", c.Name)
348+
case utils.SShKey:
349+
return fmt.Sprintf("%s-ssh-key", c.Name)
331350
case utils.XenonMetaData:
332351
return fmt.Sprintf("%s-xenon", c.Name)
333352
case utils.ConfigMap:

mysqlcluster/syncer/headless_service.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
5757
// Use `publishNotReadyAddresses` to be able to access pods even if the pod is not ready.
5858
service.Spec.PublishNotReadyAddresses = true
5959

60-
if len(service.Spec.Ports) != 2 {
61-
service.Spec.Ports = make([]corev1.ServicePort, 2)
60+
if len(service.Spec.Ports) != 3 {
61+
service.Spec.Ports = make([]corev1.ServicePort, 3)
6262
}
6363

6464
service.Spec.Ports[0].Name = utils.MysqlPortName
@@ -68,6 +68,10 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
6868
service.Spec.Ports[1].Name = utils.XBackupPortName
6969
service.Spec.Ports[1].Port = utils.XBackupPort
7070
service.Spec.Ports[1].TargetPort = intstr.FromInt(utils.XBackupPort)
71+
// ssh port
72+
service.Spec.Ports[2].Name = utils.SshPortName
73+
service.Spec.Ports[2].Port = utils.SshPort
74+
service.Spec.Ports[2].TargetPort = intstr.FromInt(utils.SshPort)
7175
return nil
7276
})
7377
}

mysqlcluster/syncer/pdb.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ package syncer
1818

1919
import (
2020
"github.com/presslabs/controller-util/pkg/syncer"
21-
policyv1beta1 "k8s.io/api/policy/v1beta1"
21+
policyv1beta1 "k8s.io/api/policy/v1"
2222
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2323
"k8s.io/apimachinery/pkg/util/intstr"
2424
"sigs.k8s.io/controller-runtime/pkg/client"

mysqlcluster/syncer/sshSecret.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
Copyright 2021 RadonDB.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package syncer
18+
19+
import (
20+
"github.com/presslabs/controller-util/pkg/syncer"
21+
"github.com/radondb/radondb-mysql-kubernetes/mysqlcluster"
22+
"github.com/radondb/radondb-mysql-kubernetes/utils"
23+
corev1 "k8s.io/api/core/v1"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"sigs.k8s.io/controller-runtime/pkg/client"
26+
)
27+
28+
// NewSecretSyncer returns secret syncer.
29+
func NewSShKeySyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
30+
secret := &corev1.Secret{
31+
TypeMeta: metav1.TypeMeta{
32+
APIVersion: "v1",
33+
Kind: "Secret",
34+
},
35+
ObjectMeta: metav1.ObjectMeta{
36+
Name: c.GetNameForResource(utils.SShKey),
37+
Namespace: c.Namespace,
38+
},
39+
}
40+
41+
return syncer.NewObjectSyncer("Secret", c.Unwrap(), secret, cli, func() error {
42+
43+
if secret.Data == nil {
44+
secret.Data = make(map[string][]byte)
45+
}
46+
47+
if len(secret.Data["id_ecdsa"]) == 0 {
48+
pub, priv, err := GenSSHKey()
49+
if err != nil {
50+
return err
51+
}
52+
secret.Data["id_ecdsa"] = priv
53+
secret.Data["authorized_keys"] = pub
54+
55+
}
56+
57+
return nil
58+
})
59+
}

mysqlcluster/syncer/sshkey.go

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
Copyright 2021 RadonDB.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package syncer
18+
19+
import (
20+
"crypto/ecdsa"
21+
"crypto/elliptic"
22+
"crypto/rand"
23+
"crypto/x509"
24+
"encoding/pem"
25+
"log"
26+
27+
"golang.org/x/crypto/ssh"
28+
)
29+
30+
// ssh -o UserKnownHostsFile=/dev/null
31+
func GenSSHKey() (pubkey, privekey []byte, err error) {
32+
33+
bitSize := 4096
34+
35+
privateKey, err := generatePrivateKey(bitSize)
36+
if err != nil {
37+
return nil, nil, err
38+
}
39+
40+
publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey)
41+
if err != nil {
42+
return nil, nil, err
43+
}
44+
45+
privateKeyBytes := encodePrivateKeyToPEM(privateKey)
46+
47+
return publicKeyBytes, privateKeyBytes, err
48+
49+
}
50+
51+
// generatePrivateKey creates a RSA Private Key of specified byte size
52+
func generatePrivateKey(bitSize int) (*ecdsa.PrivateKey, error) {
53+
// Private Key generation
54+
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
55+
if err != nil {
56+
return nil, err
57+
}
58+
59+
// Validate Private Key
60+
// err = privateKey.Validate()
61+
// if err != nil {
62+
// return nil, err
63+
// }
64+
65+
log.Println("Private Key generated")
66+
return privateKey, nil
67+
}
68+
69+
// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format
70+
func encodePrivateKeyToPEM(privateKey *ecdsa.PrivateKey) []byte {
71+
// Get ASN.1 DER format
72+
privDER, err := x509.MarshalECPrivateKey(privateKey)
73+
if err != nil {
74+
panic(err)
75+
}
76+
// pem.Block
77+
privBlock := pem.Block{
78+
Type: "EC PRIVATE KEY",
79+
Headers: nil,
80+
Bytes: privDER,
81+
}
82+
83+
// Private key in PEM format
84+
privatePEM := pem.EncodeToMemory(&privBlock)
85+
86+
return privatePEM
87+
}
88+
89+
// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file
90+
// returns in the format "ssh-rsa ..."
91+
func generatePublicKey(privatekey *ecdsa.PublicKey) ([]byte, error) {
92+
publicKey, err := ssh.NewPublicKey(privatekey)
93+
if err != nil {
94+
return nil, err
95+
}
96+
97+
pubKeyBytes := ssh.MarshalAuthorizedKey(publicKey)
98+
99+
log.Println("Public key generated")
100+
return pubKeyBytes, nil
101+
}

mysqlcluster/syncer/statefulset_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package syncer
1818

1919
import (
20+
"fmt"
2021
"testing"
2122

2223
appsv1 "k8s.io/api/apps/v1"
@@ -145,3 +146,9 @@ func TestStatefulSetSyncer_sfsUpdated(t *testing.T) {
145146
})
146147
}
147148
}
149+
150+
func TestSecretKey(t *testing.T) {
151+
pub, priv := GenSSHKey()
152+
fmt.Println(string(pub))
153+
fmt.Println(string(priv))
154+
}

0 commit comments

Comments
 (0)