VSO Event Notifications for Dynamic Secrets

[jaireddjawed]

Description

This PR adds the syncConfig.insantUpdates option to VaultDynamicSecret. When enabled, Vault will send event notifications to Vault Secrets Operator. This will be used to update on a near-immediate basis when secret information when the secret is updated in Vault. This PR adds the basic event notification skeleton code so that VSO can be aware of secret events. Responding to these secret events will be added in future PRs.

Jira Ticket

https://hashicorp.atlassian.net/browse/VAULT-40801

Testing

Set up connection to kubernetes and database secret in Vault

Set up authentication with kubernetes and created a database secret in Vault

Database Secret script file

#!/usr/bin/env bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

set -e

cat <<EOF | kubectl -n vault exec -i vault-0 -- sh -e
vault --version
vault secrets disable kvv2/
vault secrets enable -path=kvv2 kv-v2
vault kv put kvv2/secret username="db-readonly-username" password="db-secret-password"

vault secrets disable database
vault secrets enable database

vault write database/config/my-postgresql-database \
    plugin_name="postgresql-database-plugin" \
    allowed_roles="my-role" \
    connection_url="postgresql://my_database_uri" \
    username="POSGRES_USER" \
    password="POSTGRES_PASSWORD" \
    password_authentication="password"

vault write database/roles/my-role \
    db_name=my-postgresql-database \
    creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";" \
    default_ttl="10m" \
    max_ttl="10m"

cat <<EOT > /tmp/policy.hcl
path "kvv2/data/secret" {
   capabilities = ["read", "list", "subscribe"]
   subscribe_event_types = ["kv*"]
}
path "database/*" {
  capabilities = ["read", "create", "update", "subscribe"]
  subscribe_event_types=["database*"]
}
path "sys/events/subscribe/*" {
    capabilities = ["read"]
}
EOT
vault policy write demo /tmp/policy.hcl

# setup the necessary auth backend
vault auth disable kubernetes
vault auth enable kubernetes
vault write auth/kubernetes/config \
    kubernetes_host=https://kubernetes.default.svc
vault write auth/kubernetes/role/demo \
    bound_service_account_names=default \
    bound_service_account_namespaces=tenant-1,tenant-2 \
    policies=demo \
    ttl=1h
EOF

for ns in tenant-{1,2} ; do
    kubectl delete namespace --wait --timeout=30s "${ns}" &> /dev/null || true
    kubectl create namespace "${ns}"
done

Create VaultConnection Resource

Click to expand VaultConnection resources

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultConnection
metadata:
  labels:
    app.kubernetes.io/name: vaultconnection
    app.kubernetes.io/instance: vaultconnection-sample
    app.kubernetes.io/part-of: vault-secrets-operator
    app.kubernetes.io/managed-by: kustomize
    app.kubernetes.io/created-by: vault-secrets-operator
  name: vaultconnection-sample
  namespace: tenant-1
spec:
  address: http://vault.vault.svc.cluster.local:8200
  skipTLSVerify: true
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultConnection
metadata:
  labels:
    app.kubernetes.io/name: vaultconnection
    app.kubernetes.io/instance: vaultconnection-sample
    app.kubernetes.io/part-of: vault-secrets-operator
    app.kubernetes.io/managed-by: kustomize
    app.kubernetes.io/created-by: vault-secrets-operator
  name: vaultconnection-sample
  namespace: tenant-2
spec:
  address: http://vault.vault.svc.cluster.local:8200
  skipTLSVerify: true

Create VaultAuth Resource

Click to expand VaultAuth resource

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
  labels:
    app.kubernetes.io/name: vaultauth
    app.kubernetes.io/instance: vaultauth-sample
    app.kubernetes.io/part-of: vault-secrets-operator
    app.kubernetes.io/managed-by: kustomize
    app.kubernetes.io/created-by: vault-secrets-operator
  name: vaultauth-sample
  namespace: tenant-1
spec:
  vaultConnectionRef: vaultconnection-sample
  method: kubernetes
  mount: kubernetes
  kubernetes:
    role: demo
    serviceAccount: default
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
  labels:
    app.kubernetes.io/name: vaultauth
    app.kubernetes.io/instance: vaultauth-sample
    app.kubernetes.io/part-of: vault-secrets-operator
    app.kubernetes.io/managed-by: kustomize
    app.kubernetes.io/created-by: vault-secrets-operator
  name: vaultauth-sample
  namespace: tenant-2
spec:
  vaultConnectionRef: vaultconnection-sample
  method: kubernetes
  mount: kubernetes
  kubernetes:
    role: demo
    serviceAccount: default

Create VaultDynamicSecret Resource

Click to expand VaultDynamicSecret resource

# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultDynamicSecret
metadata:
  labels:
    app.kubernetes.io/name: vaultdynamicsecret
    app.kubernetes.io/instance: vaultdynamicsecret-sample
    app.kubernetes.io/part-of: vault-secrets-operator
    app.kubernetes.io/managed-by: kustomize
    app.kubernetes.io/created-by: vault-secrets-operator
  name: vaultdynamicsecret-sample
  namespace: tenant-1
spec:
  # Your existing VaultAuth (e.g., Kubernetes service account auth)
  vaultAuthRef: vaultauth-sample

  # Mount path where the database engine is enabled (e.g., 'database', 'db-prod', etc.)
  mount: database

  # Full path to your *existing* role: <engine-type>/creds/<role-name>
  # Common examples:
  #   postgresql/creds/my-app-role
  #   mysql/creds/app-readonly
  #   mongodb/creds/web-app
  path: creds/my-role # CHANGE THIS

  # How often VSO should renew (recommend: 60–80% of role's default_ttl)
  refreshAfter: 7m

  # Where to sync the credentials in Kubernetes
  destination:
    name: db-creds-dynamic # K8s Secret name your app will use
    create: true # VSO creates it if missing
    # type: Opaque              # default

  syncConfig:
    # enable event watcher to update secrets on event notification
    # so that updates are applied immediately
    instantUpdates: true

Attempted a manual rotation of the database secret and confirmed that the websocket logged the connection in the logs of VSO

│ 2025-11-19T08:02:09.395373835Z manager {"level":"info","ts":"2025-11-19T08:02:09Z","msg":"Event watcher enabled","controller":"vaultdynamicsecret","controllerGroup":"secrets.hashicorp.com","controllerKind":"VaultDyna │
│ micSecret","VaultDynamicSecret":{"name":"vaultdynamicsecret-sample","namespace":"tenant-1"},"namespace":"tenant-1","name":"vaultdynamicsecret-sample","reconcileID":"79331ec0-f4f6-41cc-b21d-05daa1880221","podUID":"113 │
│ 7a887-968f-4fe6-89ad-92b9ae4e5771"}                                                                                                                                                                                      │
│ 2025-11-19T08:02:09.395783668Z manager {"level":"info","ts":"2025-11-19T08:02:09Z","logger":"ensureEventWatcher","msg":"Starting event watcher","controller":"vaultdynamicsecret","controllerGroup":"secrets.hashicorp.c │
│ om","controllerKind":"VaultDynamicSecret","VaultDynamicSecret":{"name":"vaultdynamicsecret-sample","namespace":"tenant-1"},"namespace":"tenant-1","name":"vaultdynamicsecret-sample","reconcileID":"79331ec0-f4f6-41cc-b │
│ 21d-05daa1880221","meta":{"LastGeneration":1,"LastClientID":"521dc906a6fb947ca59473758b02a70c040495e5c501d9db15de4af982e618be"}}                                                                                         │
│ 2025-11-19T08:02:10.048468668Z manager {"level":"info","ts":"2025-11-19T08:02:10Z","logger":"streamDynamicSecretEvents","msg":"Received message","message type":"MessageText","message":{"data":{"event":{"metadata":{"p │
│ ath":"database/creds/my-role","modified":"true"}},"namespace":""}}}                                                                                                                                                      │
│ 2025-11-19T08:02:10.048832918Z manager {"level":"info","ts":"2025-11-19T08:02:10Z","logger":"streamDynamicSecretEvents","msg":"modified Event received from Vault","namespace":"","path":"database/creds/my-role","spec. │
│ namespace":""}                                                                                                                                                                                                           │
│ 2025-11-19T08:02:10.054201668Z manager {"level":"info","ts":"2025-11-19T08:02:10Z","msg":"Event watcher enabled","controller":"vaultdynamicsecret","controllerGroup":"secrets.hashicorp.com","controllerKind":"VaultDyna │
│ micSecret","VaultDynamicSecret":{"name":"vaultdynamicsecret-sample","namespace":"tenant-1"},"namespace":"tenant-1","name":"vaultdynamicsecret-sample","reconcileID":"4971de81-ea26-44f0-b73e-845ab335df20","podUID":"113 │
│ 7a887-968f-4fe6-89ad-92b9ae4e5771"}                                                                                                                                                                                      │
│ 2025-11-19T08:02:10.056046543Z manager {"level":"info","ts":"2025-11-19T08:02:10Z","logger":"getEvents","msg":"Websocket client closed, stopping GetEvents for","namespace":"tenant-1","name":"vaultdynamicsecret-sample │
│ ","err":"failed to read from websocket: failed to get reader: context canceled, message: \"\""}                                                                                                                          │
│ 2025-11-19T08:02:10.056133210Z manager {"level":"info","ts":"2025-11-19T08:02:10Z","logger":"ensureEventWatcher","msg":"Starting event watcher","controller":"vaultdynamicsecret","controllerGroup":"secrets.hashicorp.c │
│ om","controllerKind":"VaultDynamicSecret","VaultDynamicSecret":{"name":"vaultdynamicsecret-sample","namespace":"tenant-1"},"namespace":"tenant-1","name":"vaultdynamicsecret-sample","reconcileID":"4971de81-ea26-44f0-b │
│ 73e-845ab335df20","meta":{"LastGeneration":2,"LastClientID":"521dc906a6fb947ca59473758b02a70c040495e5c501d9db15de4af982e618be"}}                                                                                         │
│ 2025-11-19T08:02:40.168047877Z manager {"level":"info","ts":"2025-11-19T08:02:40Z","logger":"streamDynamicSecretEvents","msg":"Received message","message type":"MessageText","message":{"data":{"event":{"metadata":{"p │
│ ath":"database/rotate-root/my-postgresql-database","modified":"true"}},"namespace":""}}}                                                                                                                                 │
│ 2025-11-19T08:02:40.168104252Z manager {"level":"info","ts":"2025-11-19T08:02:40Z","logger":"streamDynamicSecretEvents","msg":"modified Event received from Vault","namespace":"","path":"database/rotate-root/my-postgr │
│ esql-database","spec.namespace":""}                                                                                                                                                                                      

PCI review checklist

• I have documented a clear reason for, and description of, the change I am making.

• If applicable, I've documented a plan to revert these changes if they require more than reverting the pull request.

• If applicable, I've documented the impact of any changes to security controls.

  Examples of changes to security controls include using new access control methods, adding or removing logging pipelines, etc.

[tvoran]

From the example logs it looks like this is responding to credential generation and rotate-root. I think for dynamic secrets, this should really only care about lease revocation? Like if the lease for the dynamic credentials was revoked on the Vault side.

The credentials were generated by VSO, and IIRC it would've already re-queued the VDS object based on the ttl of the lease, so I don't think VSO should respond to that event. And I'm not following why VSO would want to react to the root credentials being rotated, since those are what Vault uses to connect to the database, and not a secret VSO would have access to.