feat(ceph): Export credentials into a secret (#250)

Author: tumido

ceph/README.md

@@ -31,19 +31,19 @@ To install Ceph Object Storage add the following to the `kfctl` yaml file.
 
 * Deployment of `ceph-nano` will create a route to a `ceph-nano-dashboard` that provides a S3 web portal for the in pod object storage.
 
-* The ACCESS_KEY and SECRET_KEY created for this deployment can be retrieved from the `ceph-nano-0` pod under `/nano_user_details`
+* The ACCESS_KEY and SECRET_KEY created for this deployment can be found in `ceph-nano-credentials` secret.
   ```
-  # While logged in to the cluster and in the Open Data Hub namespace
-  # Output the ceph-nano radosgw settings
-  $ oc exec ceph-nano-0 -- cat /nano_user_details | jq '.keys'
-    ...
-        "keys": [
-          {
-            "user": "cn",
-            "access_key": "ABCDEFGHIJKL01234567",
-            "secret_key": "mnOPQRSTUVWXYZV6oSrx2MDtfEUK8R0ETagp5A9X"
-          }
-        ],
-    ...                                                       ],
+  oc describe secrets/ceph-nano-credentials
+
+  Name:         ceph-nano-credentials
+  Namespace:    odh-ceph
+  Labels:       <none>
+  Annotations:
+  Type:         Opaque
+
+  Data
+  ====
+  AWS_ACCESS_KEY_ID:      20 bytes
+  AWS_SECRET_ACCESS_KEY:  40 bytes
   ```
   ***NOTE***: The ACCESS_KEY and SECRET_KEY will change EVERY time the pod starts

ceph/object-storage/nano/base/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - service-account.yaml
 - service.yaml
 - statefulset.yaml
+- secret-rbac.yaml
 
 commonLabels:
   opendatahub.io/component: "true"
@@ -14,3 +15,5 @@ images:
 - name: ceph/daemon
   newTag: v0.7
   newName: quay.io/ceph/cn-core
+- name: oc
+  newName: quay.io/openshift/origin-cli

ceph/object-storage/nano/base/secret-rbac.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ceph-nano-secrets
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+      - get
+      - list
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ceph-nano-secrets
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ceph-nano-secrets
+subjects:
+  - kind: ServiceAccount
+    name: ceph-nano

ceph/object-storage/nano/base/statefulset.yaml

@@ -20,35 +20,82 @@ spec:
         daemon: nano
     spec:
       serviceAccountName: ceph-nano
+      initContainers:
+        - name: ceph-nano-init
+          image: ceph/daemon
+          command: ["/bin/bash", "-c", "--"]
+          args: ["cp /opt/ceph-container/tmp/* /mnt"]
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /mnt
+              name: user-details
       containers:
-      - image: ceph/daemon
-        imagePullPolicy: Always
-        name: ceph-nano
-        ports:
-        - containerPort: 8000
-        name: cn-s3
-        protocol: TCP
-        resources:
-          limits:
-            cpu: "1"
-            memory: 2G
-          requests:
-            cpu: "1"
-            memory: 512M
-        env:
-        - name: NETWORK_AUTO_DETECT
-          value: "4"
-        - name: RGW_FRONTEND_PORT
-          value: "8000"
-          # Keep this for backward compatiblity, the option is gone since https://github.com/ceph/ceph-container/pull/1356
-        - name: RGW_CIVETWEB_PORT
-          value: "8000"
-        - name: SREE_PORT
-          value: "5001"
-        - name: CEPH_DEMO_UID
-          value: "nano"
-          # Enable all of the ceph daemons to run
-        - name: CEPH_DAEMON
-          value: "demo"
-        - name: DEBUG
-          value: "verbose"
+        - image: ceph/daemon
+          imagePullPolicy: Always
+          name: ceph-nano
+          ports:
+            - containerPort: 8000
+          protocol: TCP
+          resources:
+            limits:
+              cpu: "1"
+              memory: 2G
+            requests:
+              cpu: "1"
+              memory: 512M
+          env:
+            - name: NETWORK_AUTO_DETECT
+              value: "4"
+            - name: RGW_FRONTEND_PORT
+              value: "8000"
+              # Keep this for backward compatiblity, the option is gone since https://github.com/ceph/ceph-container/pull/1356
+            - name: RGW_CIVETWEB_PORT
+              value: "8000"
+            - name: SREE_PORT
+              value: "5001"
+            - name: CEPH_DEMO_UID
+              value: "nano"
+              # Enable all of the ceph daemons to run
+            - name: CEPH_DAEMON
+              value: "demo"
+            - name: DEBUG
+              value: "verbose"
+          volumeMounts:
+            - mountPath: /opt/ceph-container/tmp
+              name: user-details
+        - image: oc
+          name: credentials-exporter
+          command: ["/bin/bash", "-c", "--"]
+          args:
+            - |
+              until [ -f /mnt/cn_user_details ]; do echo "waiting for user details file..."; sleep 10; done
+
+              oc login --token=TOKEN --server=https://openshift.default.svc.cluster.local --insecure-skip-tls-verify
+              TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              ACCESS_KEY=$(cat /mnt/cn_user_details | python -c "import sys, json; print(json.load(sys.stdin)['keys'][0]['access_key'])")
+              SECRET_KEY=$(cat /mnt/cn_user_details | python -c "import sys, json; print(json.load(sys.stdin)['keys'][0]['secret_key'])")
+
+              if [ -z "$ACCESS_KEY" ] || [ -z "$SECRET_KEY" ]; then
+                echo "Unable to parse credentials from user details!"
+                exit 1
+              fi
+
+              cat <<EOF | oc apply -f -
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: ceph-nano-credentials
+              type: Opaque
+              stringData:
+                AWS_ACCESS_KEY_ID: $ACCESS_KEY
+                AWS_SECRET_ACCESS_KEY: $SECRET_KEY
+              EOF
+
+              echo "waiting for ceph-nano to finish"
+              while true; do sleep 10; done;
+          volumeMounts:
+            - mountPath: /mnt
+              name: user-details
+      volumes:
+        - name: user-details
+          emptyDir: {}