Measuring Platform Success

deployment/addons/kubevela/templates/traits/component-policy.yaml

@@ -48,7 +48,7 @@ spec:
         			metadata: name: "\(context.appName)-\(context.name)-iam-policy"
         			spec: {
         				name: "\(context.appName)-\(context.name)-iam-policy"
-        				forProvider: policy: "\(policy)"
+        				forProvider: policy: "\(parameter.policy)"
         			}
         		}
         	}

packages/argocd/dev/appproject-modern-engg.yaml

@@ -24,6 +24,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: dapr-system
       server: https://kubernetes.default.svc
+    - namespace: devlake
+      server: https://kubernetes.default.svc
     - namespace: ingress-nginx
       server: https://kubernetes.default.svc
     - namespace: kyverno

packages/devlake/base/values.yaml

@@ -0,0 +1,82 @@
+# dependency chart values
+grafana:
+  enabled: false
+  external:
+    url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments - doesn't work - need some sort of ingress maybe?
+
+mysql:
+  useExternal: true
+  externalServer: "devlake-mysql-service"
+  externalPort: "3306"
+
+option:
+  database: mysql
+  connectionSecretName: "devlake-mysql-auth"
+  autoCreateSecret: false
+
+
+lake:
+  image:
+    repository: devlake.docker.scarf.sh/apache/devlake
+    pullPolicy: Always
+    tag: v1.0.3-beta1
+  # storage for config
+  encryptionSecret:
+    # The name of secret which contains keys named ENCRYPTION_SECRET
+    secretName: devlake-encryption-secret
+    autoCreateSecret: false
+
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1010    
+    runAsGroup: 1010   
+    fsGroup: 1010      
+
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    capabilities:
+      drop: ["ALL"]
+    seccompProfile:
+      type: RuntimeDefault
+
+
+alpine:
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    capabilities:
+      drop: ["ALL"]
+    seccompProfile:
+      type: RuntimeDefault
+
+ui:
+  image:
+    repository: devlake.docker.scarf.sh/apache/devlake-config-ui
+    pullPolicy: Always
+    tag: v1.0.3-beta1
+
+  securityContext:
+    runAsNonRoot: true
+
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    capabilities:
+      drop: ["ALL"]
+    seccompProfile:
+      type: RuntimeDefault
+
+  # Look into adding an auth proxy like here: https://github.com/oauth2-proxy/oauth2-proxy/blob/master/contrib/local-environment/docker-compose-keycloak.yaml
+
+service:
+  # service type: NodePort/ClusterIP
+  type: ClusterIP
+
+ingress:
+  enabled: false

packages/devlake/dev/external-secrets.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: devlake-encryption-secret
+  namespace: devlake 
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    name: devlake 
+    kind: SecretStore
+  target:
+    name: devlake-encryption-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: ENCRYPTION_SECRET 
+      remoteRef:
+        key: modern-engg/devlake/encryption
+        property: ENCRYPTION_SECRET 

packages/devlake/dev/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- external-secrets.yaml
+- mysql-setup-workflow.yaml
+- rds-mysql.yaml

packages/devlake/dev/mysql-init-job.yaml

@@ -0,0 +1,86 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: devlake-mysql-access-binding
+subjects:
+- kind: ServiceAccount
+  name: devlake-mysql-access
+  namespace: devlake
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mysql-user-setup
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  template:
+    metadata:
+      generateName: mysql-user-setup-
+    spec:
+      serviceAccountName: devlake-mysql-access
+      restartPolicy: OnFailure
+      initContainers:
+      - name: create-secret
+        image: bitnami/kubectl
+        command:
+          - /bin/bash
+          - -c
+          - |
+            while true; do
+                CONN_SECRET_NAME=$(kubectl get secrets -n crossplane-system -o custom-columns=NAME:.metadata.name | grep "^devlake-mysql.*cluster-mysql-connection$" || true)
+                if [ -n "$CONN_SECRET_NAME" ]; then
+                    break
+                fi
+                echo "Waiting for secret to be available..."
+                sleep 5
+            done
+            endpoint=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode)
+            kubectl create service externalname devlake-mysql-service --external-name=${endpoint} -n devlake
+            password=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode)
+            kubectl create secret generic devlake-mysql-auth -n devlake \
+            --from-literal=MYSQL_USER=merico \
+            --from-literal=MYSQL_PASSWORD=merico \
+            --from-literal=MYSQL_DATABASE=lake \
+            --from-literal=MYSQL_ROOT_PASSWORD=$password \
+            --from-literal=DB_URL="mysql://merico:merico@devlake-mysql-service:3306/lake?charset=utf8mb4&parseTime=True"
+      containers:
+      - name: create-grafana-user
+        image: mysql:8
+        command: 
+          - /bin/bash
+          - -c
+          - |
+            until mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do
+              echo "Waiting for MySQL to be ready..."
+              sleep 2
+            done
+
+            mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+            CREATE DATABASE IF NOT EXISTS lake;
+            "
+
+            echo "Creating devlake user..."
+            mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+            CREATE USER IF NOT EXISTS 'merico' IDENTIFIED BY 'merico';
+            GRANT ALL PRIVILEGES ON lake.* TO 'merico';
+            FLUSH PRIVILEGES;
+            "
+
+            echo "Creating Grafana user..."
+            mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+            CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password';
+            GRANT SELECT ON lake.* TO 'grafanaReader';
+            FLUSH PRIVILEGES;
+            "
+
+            echo "User creation completed."
+        envFrom:
+          - secretRef:
+              name: devlake-mysql-auth
+

packages/devlake/dev/mysql-setup-workflow.yaml

@@ -0,0 +1,149 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: devlake-mysql-setup-sa
+  namespace: devlake
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: devlake-mysql-setup-access-binding
+subjects:
+- kind: ServiceAccount
+  name: devlake-mysql-setup-sa
+  namespace: devlake
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: mysql-setup-workflow
+  namespace: devlake
+spec:
+  serviceAccountName: devlake-mysql-setup-sa
+  entrypoint: mysql-setup
+  templates:
+  - name: mysql-setup
+    dag:
+      tasks:
+      - name: create-secrets
+        template: create-secrets
+      - name: wait-for-mysql
+        template: wait-for-mysql
+        dependencies: [create-secrets]
+      - name: create-users
+        template: create-users
+        dependencies: [wait-for-mysql]
+
+  - name: create-secrets
+    container:
+      image: bitnami/kubectl
+      command: ["/bin/bash", "-c"]
+      args:
+      - |
+        while true; do
+            CONN_SECRET_NAME=$(kubectl get secrets -n crossplane-system -o custom-columns=NAME:.metadata.name | grep "^devlake-mysql.*cluster-mysql-connection$" || true)
+            if [ -n "$CONN_SECRET_NAME" ]; then
+                break
+            fi
+            echo "Waiting for secret to be available..."
+            sleep 10
+        done
+
+        echo "Secret found: ${CONN_SECRET_NAME}"
+        while true; do
+            endpoint=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode)
+            if [ -n "$endpoint" ]; then
+                break
+            fi
+            echo "Waiting for endpoint to be available..."
+            sleep 10
+        done
+        echo "Retrieved endpoint: $endpoint"
+
+        # Create service pointing to the RDS endpoint
+        kubectl create service externalname devlake-mysql-service --external-name=${endpoint} -n devlake --dry-run=client -o yaml | kubectl apply -f -
+
+        while true; do
+            password=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode)
+            if [ -n "$password" ]; then
+                break
+            fi
+            echo "Waiting for password to be available..."
+            sleep 10
+        done
+        echo "Password retrieved (first 2 chars): ${password:0:2}..."
+
+        # Create auth secret with credentials
+        kubectl create secret generic devlake-mysql-auth -n devlake \
+        --from-literal=MYSQL_USER=merico \
+        --from-literal=MYSQL_PASSWORD=merico \
+        --from-literal=MYSQL_DATABASE=lake \
+        --from-literal=MYSQL_ROOT_PASSWORD=$password \
+        --from-literal=DB_URL="mysql://merico:merico@devlake-mysql-service:3306/lake?charset=utf8mb4&parseTime=True" --dry-run=client -o yaml | kubectl apply -f -
+
+  - name: wait-for-mysql
+    retryStrategy:
+      limit: 40
+      retryPolicy: "Always"
+      backoff:
+        duration: "10"
+        factor: 2
+        maxDuration: "300s"
+    container:
+      image: mysql:8
+      command: ["/bin/bash", "-c"]
+      args:
+      - |
+        echo "Waiting for MySQL to be fully initialized..."
+        sleep 60  # Initial delay to allow RDS to initialize
+
+        if mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; then
+          echo "MySQL is ready!"
+          exit 0
+        else
+          echo "MySQL not ready yet, will retry..."
+          exit 1
+        fi
+      env:
+      - name: MYSQL_ROOT_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: devlake-mysql-auth
+            key: MYSQL_ROOT_PASSWORD
+
+  - name: create-users
+    container:
+      image: mysql:8
+      command: ["/bin/bash", "-c"]
+      args:
+      - |
+        echo "Creating lake database..."
+        mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+        CREATE DATABASE IF NOT EXISTS lake;
+        "
+
+        echo "Creating devlake user..."
+        mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+        CREATE USER IF NOT EXISTS 'merico' IDENTIFIED BY 'merico';
+        GRANT ALL PRIVILEGES ON lake.* TO 'merico';
+        FLUSH PRIVILEGES;
+        "
+
+        echo "Creating Grafana user..."
+        mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "
+        CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password';
+        GRANT SELECT ON lake.* TO 'grafanaReader';
+        FLUSH PRIVILEGES;
+        "
+
+        echo "User creation completed."
+      env:
+      - name: MYSQL_ROOT_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: devlake-mysql-auth
+            key: MYSQL_ROOT_PASSWORD