diff --git a/.github/ct.yaml b/.github/ct.yaml index 6d923e768..e11e4a5fc 100644 --- a/.github/ct.yaml +++ b/.github/ct.yaml @@ -1,4 +1,2 @@ -chart-repos: - - bitnami=https://charts.bitnami.com/bitnami target-branch: main check-version-increment: false diff --git a/.github/workflows/helm-chart-lint-test.yaml b/.github/workflows/helm-chart-lint-test.yaml index 104037db5..2b1b94223 100644 --- a/.github/workflows/helm-chart-lint-test.yaml +++ b/.github/workflows/helm-chart-lint-test.yaml @@ -42,6 +42,44 @@ jobs: if: steps.list-changed.outputs.changed == 'true' run: ct lint --config .github/ct.yaml + # `helm lint` does NOT enforce the chart's own guards. On Helm 4 a + # template `fail` surfaces as `level=INFO msg="funcMap fail"` and lint + # still exits 0 -- verified on Helm 4.2.3. So a values file carrying stale + # Bitnami keys, or a Bitnami image, would pass `ct lint` untouched. + # `helm template` does fail, so gate on that. + - name: Render guards (helm template, which lint cannot enforce) + if: steps.list-changed.outputs.changed == 'true' + run: | + set -euo pipefail + helm template ci charts/nebraska > /dev/null + helm template ci charts/nebraska \ + --set postgresql.primary.persistence.enabled=true > /dev/null + + # Each of these MUST fail. If one starts passing, a guard has regressed. + for bad in \ + "postgresql.metrics.enabled=true" \ + "postgresql.image.repository=bitnamilegacy/postgresql" \ + "postgresql.primary.resources.limits.memory=4Gi" ; do + if helm template ci charts/nebraska --set "$bad" > /dev/null 2>&1; then + echo "::error::guard regression: --set $bad should have been rejected" + exit 1 + fi + done + + # The data-directory guard is upgrade-only. + if helm template ci charts/nebraska --is-upgrade \ + --set postgresql.primary.persistence.enabled=true > /dev/null 2>&1; then + echo "::error::guard regression: persistent upgrade without acknowledgement was allowed" + exit 1 + fi + + # Values left at Bitnami defaults must NOT be rejected, so that + # vendoring the old values.yaml wholesale still installs. + helm template ci charts/nebraska \ + --set postgresql.architecture=standalone \ + --set postgresql.metrics.enabled=false \ + --set postgresql.tls.enabled=false > /dev/null + - name: Create kind cluster uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 if: steps.list-changed.outputs.changed == 'true' diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index 183bfb850..fc92fc792 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -32,10 +32,6 @@ jobs: with: version: v3.12.1 - - name: Add Helm repos - run: | - helm repo add bitnami https://charts.bitnami.com/bitnami - - name: Run chart-releaser uses: helm/chart-releaser-action@cae68fefc6b5f367a0275617c9f83181ba54714f # v1.7.0 env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a87f0f14..09f402566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed +- **helm/postgresql: the default superuser password is now generated, not `changeIt`.** `postgresql.auth.postgresPassword` defaults to `""` and the chart generates a random 24-character password on first install, keeping any value already in the cluster across upgrades. This restores what the Bitnami subchart did before chart 2.0.0 replaced it with a fixed default. Retrieve it with `kubectl get secret -postgresql -o jsonpath='{.data.postgres-password}' | base64 -d`. Anyone relying on the published default must set `postgresql.auth.postgresPassword` explicitly or use `postgresql.auth.existingSecret`. +- **helm/postgresql: unknown `postgresql.*` values are now rejected at render time instead of being silently ignored.** Removing the subchart dropped roughly 55 keys; a values file carrying them would previously have installed cleanly with the settings discarded. Expect around twenty reports on the first upgrade if you vendored the upstream Bitnami `values.yaml`; each names the replacement key. Values left at their Bitnami defaults are accepted silently. + +- **helm: replaced the Bitnami PostgreSQL subchart with an in-chart StatefulSet on the official `postgres` image.** The chart no longer depends on `https://charts.bitnami.com/bitnami` and no longer ships a `bitnamilegacy/*` image, so the bundled database gets security patches again. This is a **breaking change for installs with `postgresql.primary.persistence.enabled: true`**: the data directory moves inside the volume. You have two options. If you stay on the same PostgreSQL major version you can reuse the volume in place, no dump needed, see the tested procedure in the chart README "Upgrading to 3.0.0". Otherwise, and always across major versions, use `pg_dump` and restore. Nebraska itself is restarted by the upgrade, through a pod template annotation, so it runs its schema migrations again against the database it now points at. Installs using an external database (`postgresql.enabled: false`) or the default ephemeral database need no action. Chart version bumped to 3.0.0. ([#1574](https://github.com/flatcar/nebraska/issues/1574), [#1148](https://github.com/flatcar/nebraska/issues/1148)) + - **Per-group runtime state moved to node-local `group_local` sidecar:** `rollout_in_progress` plus a nullable override column for each `policy_*` column on `groups` now live on a new `group_local` table, in preparation for the distributed Nebraska topology described in [RFC #1375](https://github.com/flatcar/nebraska/issues/1375). The safe-mode auto-pause brake writes the local override instead of mutating the admin default; reads return `COALESCE(override, default)`. The JSON contract is unchanged. ([#1396](https://github.com/flatcar/nebraska/pull/1396)) - **Activity events split across runtime-local and admin tables:** Admin-originated activity events (channel package updates) are now stored in a separate `admin_activity` table, in preparation for the distributed Nebraska topology described in [RFC #1375](https://github.com/flatcar/nebraska/issues/1375). The JSON contract is unchanged. ([#1398](https://github.com/flatcar/nebraska/pull/1398)) - **Package Management UI Improvements:** @@ -24,6 +29,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Channel edit dialog filters out blacklisted packages from selection - Floor package selection prevents choosing blacklisted packages with clear visual feedback ### Removed + +- **helm/postgresql: the `password` key is removed from the `-postgresql` Secret.** The Bitnami subchart emitted both `postgres-password` (superuser) and `password` (a separate app user); this chart has a single superuser and emits only `postgres-password`. Helm deletes the missing key on upgrade, so anything reading it, for example backup jobs or external tooling, must be repointed **before** upgrading. + ### Bugfixes - Fixed package blacklist changes not appearing in UI immediately after save diff --git a/charts/nebraska/Chart.lock b/charts/nebraska/Chart.lock deleted file mode 100644 index 47e72c35b..000000000 --- a/charts/nebraska/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 11.9.1 -digest: sha256:fa46eb2489fd385d4cc5bae13f161897a8bff0fb0a281f42cd17953c735ff3fe -generated: "2022-09-24T19:09:15.078346+02:00" diff --git a/charts/nebraska/Chart.yaml b/charts/nebraska/Chart.yaml index 2cc997247..5a507633c 100644 --- a/charts/nebraska/Chart.yaml +++ b/charts/nebraska/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nebraska description: Nebraska is an update manager for Flatcar Container Linux. type: application -home: https://github.com/flatcar/nebraska/tree/main/deploy/helm +home: https://github.com/flatcar/nebraska/tree/main/charts/nebraska icon: https://raw.githubusercontent.com/flatcar/nebraska/main/docs/nebraska-logo.svg keywords: - nebraska @@ -18,11 +18,33 @@ sources: maintainers: - name: flatcar url: https://flatcar.org/ -version: 2.0.0 + +# 2.0.0 -> 3.0.0: MAJOR, because the bundled PostgreSQL changed image, data +# directory layout and uid. Installs with persistence enabled need a dump and +# restore; see "Upgrading to 3.0.0" in README.md. +version: 3.0.0 appVersion: "3.0.0" -dependencies: - - name: postgresql - version: 11.9.1 - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled +# There is deliberately no `dependencies:` block, and Chart.lock is deleted. +# +# 2.0.0 depended on postgresql 11.9.1 from https://charts.bitnami.com/bitnami. +# Bitnami retired its free catalogue on 2025-08-28: images moved to the frozen +# docker.io/bitnamilegacy archive (never rebuilt, so every CVE since then is +# unpatched) and the classic chart repo is deprecated with no committed shutdown +# date. Those are flatcar/nebraska#1574 and #1148 respectively, and they are the +# same root cause, so this chart fixes them together by vendoring a small +# StatefulSet instead of swapping one external dependency for another. +# +# Alternatives considered: +# - Keep the subchart, point image.repository at docker.io/postgres. Does not +# work: the Bitnami templates set POSTGRESQL_* env vars, mount /bitnami, and +# run a Bitnami-specific entrypoint. +# - Adopt a maintained third-party postgres subchart (CloudPirates is the one +# the ecosystem converged on; Superset, docker-selenium and Opik all moved +# to it). Rejected here to avoid trading one external chart +# dependency for another for what is only a convenience database. +# - Require an operator such as CloudNativePG. Rejected as a default: it needs +# cluster-scoped CRDs installed before this chart can render at all. It +# remains the recommendation for production, via postgresql.enabled=false. +# +# Vendoring is what the closest comparable migration (helixml/helix#1890) did. diff --git a/charts/nebraska/README.md b/charts/nebraska/README.md index 3ec7abf5d..2cc9742da 100644 --- a/charts/nebraska/README.md +++ b/charts/nebraska/README.md @@ -9,6 +9,477 @@ $ helm repo add nebraska https://flatcar.github.io/nebraska $ helm install my-nebraska nebraska/nebraska ``` +## Upgrading to 3.0.0 + +**Breaking change: the bundled PostgreSQL is no longer the Bitnami subchart.** + +Chart 3.0.0 removes the `bitnami/postgresql` dependency and replaces it with a +small PostgreSQL StatefulSet defined inside this chart, running the official +`docker.io/postgres` image. + +### Why + +Bitnami retired its free catalogue on 2025-08-28. Chart 2.0.0 pinned the image +to `docker.io/bitnamilegacy/postgresql:17.5.0` to keep installs working +(flatcar/nebraska#1227), but the `bitnamilegacy` registry is a frozen archive: +it is never rebuilt, so every PostgreSQL and OS-package CVE published since then +is unpatched in every install running chart defaults (flatcar/nebraska#1574). +The chart also still depended on `https://charts.bitnami.com/bitnami`, a +deprecated endpoint with no committed shutdown date (flatcar/nebraska#1148). +Vendoring the StatefulSet resolves both: the chart now has no external chart +dependency and no Bitnami-family image. + +### Does this affect me? + +| If you... | Then... | +|-----------|---------| +| set `postgresql.enabled: false` and use an external database | **No action needed.** Nothing in this change touches you. | +| run with the default `postgresql.primary.persistence.enabled: false` | **No action needed.** Your database is already ephemeral. The bundled PostgreSQL comes back empty and the chart rolls Nebraska so it recreates its schema unattended. | +| run with `postgresql.primary.persistence.enabled: true` | **Action required, dump and restore.** See below. Reusing the volume in place is being investigated but is not yet a supported path. | + +### Why the documented path is dump and restore + +The volume itself is not the obstacle. Both chart versions mount the *same* PVC +(`data--postgresql-0`), and `postgresql.dataMountPath` / +`postgresql.dataSubdir` can point `PGDATA` at the Bitnami directory, so an +in-place reuse is mechanically expressible in this chart. What follows is an +honest accounting of what actually stands in the way. + +**Precondition for any in-place reuse: the same PostgreSQL major version.** +Chart 2.0.0 defaulted to `bitnamilegacy/postgresql:17.5.0` and the default here +is `postgres:17-bookworm`, so the on-disk format matches for anyone on the old +defaults. If you pinned an older major (the subchart's own default was 14.5.0), +you need `pg_upgrade` or a dump/restore regardless, nothing below applies. + +1. **Different data directory, configurable, not a blocker.** Bitnami stored + the cluster at `/bitnami/postgresql/data`; the default here is + `/var/lib/postgresql/data/pgdata`. Both halves are settable + (`postgresql.dataMountPath`, `postgresql.dataSubdir`). +2. **Different uid, configurable, not a blocker.** Bitnami ran as uid 1001; the + default official image here (`17-bookworm`, Debian) runs as uid 999, and the + Alpine variants use 70. `postgresql.podSecurityContext` sets all three, and + the chart already mounts `/tmp` so the entrypoint works under a uid the image + does not know. +3. **The volume contains no `postgresql.conf`.** This is the real one. Bitnami + kept its server config inside the *image* at `/opt/bitnami/postgresql/conf/` + and passed it with `--config-file`, and its entrypoint deleted + `postgresql.conf` and `pg_hba.conf` from the data directory on every start. + The official image expects both inside `PGDATA`, so pointing it at a Bitnami + volume fails with `could not access the server configuration file`. Supplying + them works: an init container that writes a minimal `postgresql.conf` / + `pg_hba.conf` into PGDATA once **is tested end to end** (2026-08-17, kind: + postgres recovered from the Bitnami WAL and served the existing database, + Nebraska reconnected without a restart). The procedure is below; dump/restore + remains the guaranteed fallback, and the only path across major versions. + +### In-place upgrade (same major version, tested) + +No dump, no restore, no PVC deletion. The StatefulSet's name, selector, service +name and claim name are identical, so the same volume is attached; three value +changes make the old data directory usable. Set these in your values file and +run `helm upgrade` with `postgresql.acknowledgeDataDirMigration=true`: + +```yaml +postgresql: + dataMountPath: /bitnami/postgresql # mount the PVC where Bitnami mounted it + dataSubdir: data # PGDATA = /bitnami/postgresql/data + podSecurityContext: # every file on the volume is uid/gid 1001 + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: { type: RuntimeDefault } + extraPodSpec: + initContainers: # one-time config bootstrap, see reason 3 + - name: pgconf-bootstrap + image: docker.io/postgres:17-bookworm + command: [/bin/sh, -c] + args: + - | + set -e + cd /bitnami/postgresql/data + [ -f postgresql.conf ] || printf "listen_addresses = '*'\n" > postgresql.conf + [ -f pg_hba.conf ] || printf "local all all trust\nhost all all 127.0.0.1/32 trust\nhost all all ::1/128 trust\nhost all all all scram-sha-256\n" > pg_hba.conf + [ -f pg_ident.conf ] || echo "# no mappings" > pg_ident.conf + securityContext: { runAsUser: 1001 } + volumeMounts: + - name: data + mountPath: /bitnami/postgresql +``` + +Caveats, all verified in the same run: + +* **Snapshot or retain the PV first.** In-place means PostgreSQL writes to the + original directory, the safety net is the volume, not an untouched `data/`. +* Expect a short WAL redo on first start (the 2.0.0 pod had no preStop hook, so + it was SIGKILLed), normal crash recovery, not corruption. +* The first start may log `chmod: changing permissions of '/var/run/postgresql': + Operation not permitted`, cosmetic, the server continues. +* **Check how your passwords are hashed before you use the `pg_hba.conf` above.** + It ends with `scram-sha-256`, which is correct for PostgreSQL 14 and newer, so + it matches a 17.5 Bitnami install. If your cluster was upgraded from an older + major version, the stored passwords may still be md5, and then every login + fails with a confusing error. To check, before you start: + + ```console + $ kubectl exec -- psql -U postgres -tAc \ + "select distinct substring(rolpassword for 4) from pg_authid where rolpassword is not null" + ``` + + `SCRAM` means you can keep the line as it is. `md5` means change that last + line to `md5`, or reset the password after the upgrade. +* This recreates no config you had under Bitnami beyond the defaults above. If + you relied on custom Bitnami settings (`shared_preload_libraries`, custom + `pg_hba` rules), port them yourself. Note the official image does not ship + `pgaudit`. +* After it works you may drop the init container and the `/bitnami` values on a + later change, or leave them, the config now lives in PGDATA where the + official image expects it. + +**Note on collation:** the default `17-bookworm` carries glibc 2.36, the same C +library the Bitnami image shipped, so sort order and btree index ordering are +unchanged by this migration. Collation only becomes a concern if you switch to an +Alpine tag (`postgres:17-alpine` is musl), where a btree index on `text`/`varchar` +built under one collation is silently wrong under another and queries can fail to +find rows that are present, see +[Locale data changes](https://wiki.postgresql.org/wiki/Locale_data_changes). +`pg_dump`/restore is explicitly *not* affected, which is one reason it is the +supported path. + +Separately, and regardless of migration path: the Bitnami chart set +`shared_preload_libraries = 'pgaudit'`, and `pgaudit` is not present in the +official image. Because that setting lived in the image's config file rather +than on the volume, it does not block anything, but **if you rely on audit +logging today, it goes away with this upgrade.** Use an image that ships the +extension if you need it. + +### Migration (persistence enabled) + +Do **not** run `helm upgrade` first, the dump has to come out of the old pod. + +```console +# 1. Stop writes. +$ kubectl scale --replicas=0 deployment/my-nebraska + +# 2. Dump from the still-running Bitnami pod. Use -U/-d explicitly: a wrong +# name produces a valid-looking but empty dump. +$ PGPW=$(kubectl get secret my-nebraska-postgresql -o jsonpath='{.data.postgres-password}' | base64 -d) +$ kubectl exec my-nebraska-postgresql-0, env PGPASSWORD="$PGPW" pg_dump -U postgres -d nebraska > nebraska.sql +$ grep -c '^COPY public\.' nebraska.sql # must be >0; `ls -l` cannot detect an empty dump + +# 3. Retain the volume BEFORE deleting anything, so a bad dump is survivable. +# Most StorageClasses use reclaimPolicy: Delete, which destroys the disk with +# the PVC. +$ PV=$(kubectl get pvc data-my-nebraska-postgresql-0 -o jsonpath='{.spec.volumeName}') +$ kubectl patch pv "$PV" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + +# 4. Delete the old StatefulSet and PVC. Note this chart deliberately keeps the +# StatefulSet's immutable fields (selector, serviceName, volumeClaimTemplates) +# identical to the Bitnami subchart's, so an in-place `helm upgrade` is NOT +# rejected by Kubernetes, it would succeed and silently start an empty +# database. That is why the chart refuses to render without +# postgresql.acknowledgeDataDirMigration=true. +$ kubectl delete statefulset my-nebraska-postgresql --cascade=orphan +$ kubectl delete pod my-nebraska-postgresql-0 +$ kubectl delete pvc data-my-nebraska-postgresql-0 + +# 5. Upgrade, keeping Nebraska scaled to zero. Without --set replicaCount=0 the +# upgrade scales Nebraska straight back up against the new EMPTY database; it +# would run its migrations and bootstrap rows, and the restore in step 7 would +# then collide with them. +# +# PASS YOUR OWN VALUES FILE. `helm upgrade` resets values to chart defaults +# unless you supply them again, and 3.0.0 defaults persistence to false, so +# omitting -f here renders PostgreSQL with no PVC at all and you would restore +# the dump into an emptyDir that disappears on the next restart. +# Do NOT use --reuse-values: it would resurrect the 2.0.0 bitnamilegacy image. +$ helm upgrade my-nebraska nebraska/nebraska --version 3.0.0 \ + -f my-values.yaml \ + --set replicaCount=0 \ + --set postgresql.primary.persistence.enabled=true \ + --set postgresql.acknowledgeDataDirMigration=true \ + --wait --timeout 10m + +# 6. Wait for the new database to be ready. +$ kubectl wait --for=condition=Ready pod/my-nebraska-postgresql-0 --timeout=300s + +# 7. Restore. ON_ERROR_STOP + single-transaction means a partial restore rolls +# back instead of leaving a half-populated database. +$ kubectl exec -i my-nebraska-postgresql-0, \ + env PGPASSWORD="$PGPW" psql -U postgres -d nebraska \ + -v ON_ERROR_STOP=1 --single-transaction < nebraska.sql + +# 8. Refresh planner statistics. A plain SQL restore does not do this, and +# without it the first queries run against empty stats. +$ kubectl exec -i my-nebraska-postgresql-0, \ + env PGPASSWORD="$PGPW" psql -U postgres -d nebraska -c 'ANALYZE' + +# 9. Verify BEFORE scaling Nebraska back up. +$ kubectl exec -i my-nebraska-postgresql-0, \ + env PGPASSWORD="$PGPW" psql -U postgres -d nebraska -tAc \ + 'select (select count(*) from application) as apps, + (select count(*) from groups) as groups, + (select count(*) from package) as packages' +# Compare against the same query run before the upgrade. + +# 10. Scale Nebraska back up. +$ helm upgrade my-nebraska nebraska/nebraska --version 3.0.0 \ + -f my-values.yaml \ + --set postgresql.primary.persistence.enabled=true \ + --set postgresql.acknowledgeDataDirMigration=true +``` + +Keep `nebraska.sql` until you have confirmed the new instance is serving +correctly, and only then remove the retained PV. + +**If you upgraded by accident and lost your data:** don't delete anything. The +Bitnami cluster is still on the volume in the `data/` directory, untouched. The +new empty cluster was created next to it in `pgdata/`. Run +`helm rollback my-nebraska` immediately and it comes back. `helm rollback` +replays the stored 2.0.0 manifest and does not re-resolve the Bitnami chart +repository, so it works even though that repository is deprecated. + +> **Helm 4 caveat.** If the Secret was ever edited outside Helm, a +> `kubectl patch` to rotate the password, or an operator writing into it, the +> rollback fails on Helm 4 with +> `conflict with "kubectl-patch" using v1: .data.postgres-password`, and leaves +> the release `failed`. Helm 4 defaults to server-side apply and will not take +> ownership of a field another manager set. Retry with: +> +> ```console +> $ helm rollback my-nebraska 1 --force-conflicts +> ``` +> +> Verified on a live cluster: the rollback then succeeds and the data comes +> back. Helm 3.12.1 (what CI uses) applies client-side and is unaffected. + +Be aware the symptom is not obvious. Nebraska only runs its schema migrations at +process start, and its liveness probe does not touch the database, so a Nebraska +pod that came up against the wrong database stays `Ready` with its probe green +while the API returns errors (`relation "application" does not exist` in the +logs). Chart 3.0.0 therefore carries a pod-template annotation +(`nebraska.flatcar.org/bundled-db-generation`) that rolls Nebraska once on +upgrade, so it re-runs its migrations against whatever database it now points +at. Do not treat a Ready pod as proof that the upgrade went well. Run the +verification query. + +### GitOps (Argo CD, Flux) + +Two things to know: + +* The chart preserves an existing password by reading the live Secret. That + lookup returns nothing during a dry-run or a bare `helm template`, so a + rendered manifest shows a *freshly generated* password on every render. This + is worse than cosmetic drift: if your tooling *applies* that manifest, the + cluster Secret is overwritten while PostgreSQL keeps the password it was + initialised with, the next pod restart fails authentication. Use + `postgresql.auth.existingSecret` with a secret you manage (SOPS, External + Secrets, Sealed Secrets) and the chart will not render a Secret at all. + Renaming `secretKeys.adminPasswordKey` on an existing install has the same + lockout effect, because the lookup only reads the new key. +* The `postgresql.acknowledgeDataDirMigration` gate only fires on a real + `helm upgrade`. Template-rendering workflows never trigger it, so if you are + moving a persistent install from 2.0.0 to 3.0.0 under GitOps, do the dump and + restore deliberately, nothing will stop you. + +### Values that changed + +| 2.0.0 | 3.0.0 | Note | +|-------|-------|------| +| `postgresql.image.repository: bitnamilegacy/postgresql` | `postgresql.image.repository: postgres` | | +| `postgresql.image.tag: 17.5.0` | `postgresql.image.tag: 17-bookworm` | Same PostgreSQL major version, and the same glibc as the Bitnami image, so collation is unchanged. | +| *(n/a)* | `postgresql.auth.existingSecret` | New: bring your own secret. | +| *(n/a)* | `postgresql.auth.secretKeys.adminPasswordKey` | New; defaults to the previous key name `postgres-password`. | +| *(n/a)* | `postgresql.dataMountPath`, `postgresql.dataSubdir` | New; see below. | +| *(n/a)* | `postgresql.args` | New: arguments for the postgres server. The only declarative, first-boot way to set start-time settings such as `wal_level` or `log_connections` (`ALTER SYSTEM` works but needs pod access and a restart). | +| *(n/a)* | `postgresql.image.digest` | New: pin the image by content rather than by tag. | +| *(n/a)* | `postgresql.startupProbe`, `postgresql.shmSizeLimit`, `postgresql.extraPodSpec` | New; see `values.yaml`. | +| *(n/a)* | `postgresql.podSecurityContext`, `postgresql.containerSecurityContext`, `postgresql.resources`, `postgresql.extraEnv`, `postgresql.extraVolumes`, `postgresql.extraVolumeMounts` | New; previously supplied by the subchart under `postgresql.primary.*`. Scheduling fields (`nodeSelector`, `tolerations`, `affinity`, `priorityClassName`) are set through `postgresql.extraPodSpec` rather than one key each. | +| any other `postgresql.*` key from the Bitnami subchart | **rejected at render time** | The chart reports any value it does not read rather than ignoring it. Keys switched off or left empty (`metrics.enabled: false`, `tls: {}`, `architecture: standalone`) are accepted silently. Keys carrying a real value are reported with the setting they moved to, see below. | + +If you vendored the upstream Bitnami `values.yaml` wholesale, expect roughly +twenty reports on the first upgrade. That is intentional: about half of them +(`primary.resources`, `primary.podSecurityContext`, `primary.persistence.mountPath`) +carry real configuration that simply moved, and silently dropping your resource +limits or your `fsGroup` is exactly the failure this guard exists to prevent. +Each message names the replacement key. It is a one-time cleanup, and the values +that genuinely did nothing are already ignored for you. + +Reported settings that are easy to miss, because they sit under keys this chart +*does* read: + +* `postgresql.serviceAccount.annotations`, not applied to the PostgreSQL + ServiceAccount; use the top-level `extraAnnotations`, which reach every object. +* `postgresql.primary.livenessProbe` / `readinessProbe`, including an explicit + `enabled: false`. Probes are fixed by this chart; `postgresql.startupProbe` + tunes the first-start budget. +* `postgresql.extraPodSpec.containers` / `volumes`, these would duplicate a key + the chart renders itself and produce an invalid pod spec. Use `postgresql.extraEnv`, + `extraVolumes` and `extraVolumeMounts`, or `extraPodSpec.initContainers`. + +`postgresql.enabled`, `postgresql.auth.username`, `postgresql.auth.database`, +`postgresql.auth.postgresPassword`, `postgresql.primary.persistence.*`, +`postgresql.serviceAccount.*` and `postgresql.nameOverride` keep their previous +names and meaning. Object names (`-postgresql`, +`-postgresql-hl`), the secret key `postgres-password` and the +`config.database.*` contract are all unchanged, so an external secret manager or +a `config.database.passwordExistingSecret` pointing at them keeps working. + +### Other behaviour changes + +* **The secret has one key, not two.** The Bitnami subchart emitted both + `postgres-password` and `password`. Only `postgres-password` is now produced. + Note the `password` key is actively **removed** from the existing Secret on + upgrade, not merely left unused, `Secret.data` has no merge patch strategy, + so Helm nulls the absent key. If you referenced `password` from your own + manifests, repoint them *before* upgrading. +* **An existing password is preserved.** If the Secret already exists in the + cluster, its current value wins over `postgresql.auth.postgresPassword`. A + password you rotated by hand is not reverted to the chart default by a later + `helm upgrade`. +* **`postgresql.auth.username` now actually works.** Under the Bitnami subchart + a non-`postgres` username created a *non-superuser* whose password lived under + the `password` key, while the chart went on connecting with the + `postgres-password` value, so anything other than `postgres` was broken. With + the official image `POSTGRES_USER` *is* the superuser initdb creates, and its + password is the one in `postgres-password`. Note this also means the chart no + longer offers a way to run Nebraska as a least-privilege, non-superuser role. +* **No `pgaudit`, and no connection logging by default.** Set + `postgresql.args: [postgres, -c, log_connections=on, -c, log_disconnections=on]` + if you need an access trail. +* **Sort order is unchanged by default, but watch it if you switch to Alpine.** + The default `17-bookworm` carries the same glibc 2.36 as the Bitnami image, so + collation, and therefore `ORDER BY` on text and btree index ordering, is + identical. Only if you set `postgresql.image.tag` to an Alpine variant does + this change: musl collation is effectively byte order, and you must move + `runAsUser`/`runAsGroup`/`fsGroup` to `70` at the same time. +* **Clean shutdowns.** The pod now has a `preStop` hook running + `pg_ctl -m fast`. Kubernetes sends SIGTERM, which PostgreSQL reads as "smart + shutdown" and which makes it wait indefinitely for Nebraska's pooled + connections to close; previously the pod was SIGKILLed at the end of the grace + period and the next start did crash recovery. +* **`readOnlyRootFilesystem: true` by default,** with `emptyDir`s at + `/var/run/postgresql` (the socket directory. PostgreSQL will not start + without it), `/tmp` and `/dev/shm`. This stops an attacker tampering with the + binaries; it does not stop code execution, because those mounts are writable + and a PostgreSQL superuser has `COPY ... TO PROGRAM` regardless. +* **The metrics exporter and the volumePermissions init container are gone.** + The Bitnami subchart could run a `postgres-exporter` sidecar with a + ServiceMonitor and PrometheusRule (`metrics.enabled`), and a root init + container that chowned the volume (`volumePermissions.enabled`). Neither is + rendered here, and setting either value is refused at render time rather than + ignored. The routes: run postgres-exporter as its own Deployment against the + PostgreSQL Service and supply it plus any ServiceMonitor through the top-level + `extraObjects`; reproduce the chown with `postgresql.extraPodSpec.initContainers` + (worked example in `values.yaml`) if your storage ignores `fsGroup`, which is + mainly some NFS provisioners. Note both of the images those features used moved + to `bitnamilegacy` and no longer pull. +* **Do not upgrade with `--force-replace`** (Helm 3's `--force`). It deletes and + recreates the Services, which changes the ClusterIP and breaks every pooled + connection Nebraska is holding. + +### Security notes + +* **The superuser password is generated on first install** and preserved across + upgrades. Retrieve it with: + ```console + $ kubectl get secret my-nebraska-postgresql -o jsonpath='{.data.postgres-password}' | base64 -d + ``` + Chart 2.0.0 shipped a fixed default of `changeIt`; that is gone. Set + `postgresql.auth.postgresPassword`, or `postgresql.auth.existingSecret`, if you + manage credentials yourself. +* **No NetworkPolicy is rendered**, matching the Bitnami subchart's default. The + database is a ClusterIP Service, so anything on the pod network can reach port + 5432. It just needs the password now, instead of a published default. To + restrict it, add one through `extraObjects`: + ```yaml + extraObjects: + - apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: '{{ .Release.Name }}-postgresql' + namespace: '{{ .Release.Namespace }}' + spec: + podSelector: + matchLabels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: '{{ .Release.Name }}' + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: nebraska + app.kubernetes.io/instance: '{{ .Release.Name }}' + ports: + - port: 5432 + protocol: TCP + ``` + Remember to allow any backup jobs as well. Requires a CNI that enforces + NetworkPolicy. +* **No memory limit is set by default**, as in chart 2.0.0 and the Bitnami + subchart, a wrong limit OOM-kills a database mid-transaction, so the chart + will not guess one. Set `postgresql.resources.limits` once you know your + working set. `/dev/shm` is bounded at 256Mi by default, which removes the + node-pressure vector that an unbounded memory-backed volume would create. +* Traffic to the database is unencrypted by default (`sslmode=disable`). To + enable TLS, mount a certificate with `postgresql.extraVolumes` and set + `postgresql.args: [postgres, -c, ssl=on, -c, ssl_cert_file=..., -c, ssl_key_file=...]`, + then set `config.database.sslMode: verify-full`. +* `postgresql.image.tag` is a floating tag pulled with `IfNotPresent`, so a node + that has already cached it will not pick up a rebuilt image. Set + `postgresql.image.digest` to pin by content, and keep it updated. + +### Is the bundled database production-ready? + +No, and it is not meant to be. It is a single replica with no backups, no +failover and no automated major-version upgrades, the same scope the Bitnami +subchart had in this chart. It exists so that `helm install` produces a working +Nebraska. + +For anything you care about, set `postgresql.enabled: false` and point +`config.database.*` at a database you operate, or at an operator such as +[CloudNativePG](https://cloudnative-pg.io/). + +That applies with particular force to the distributed topology in +[RFC #1375](https://github.com/flatcar/nebraska/issues/1375). That design gives +each region its own **writable** database kept in sync by one-way *logical* +replication, with least-privilege roles separating the admin and runtime write +surfaces. The bundled StatefulSet can technically participate, set +`postgresql.args: [postgres, -c, wal_level=logical]` and persistence on, but it +provisions no roles, manages no publications or subscriptions, and defaults to +ephemeral storage, which would destroy replication slots on every pod +replacement. An operator is the right tool there. Streaming replication and read +replicas are deliberately not offered: every Omaha check-in writes, and Nebraska +opens a single connection pool, so a read-only standby has nowhere to send +traffic. + +### Backups + +Anything that backs up over the network, `pg_dump`/`pg_dumpall` against the +`-postgresql` Service, is unaffected. The wire protocol, port, Service +name, database name and credentials are all unchanged. + +Two things do change: + +* **Volume-snapshot backups are not portable across this upgrade.** A snapshot + taken from the Bitnami PVC cannot be restored into the new StatefulSet, for + the same four reasons listed above. Take a logical dump before upgrading, and + treat any pre-upgrade snapshots as restorable only onto chart 2.0.0. +* **Backup scripts that source the Bitnami environment break.** The official + image has no `POSTGRESQL_*` variables (`POSTGRESQL_PASSWORD`, + `POSTGRESQL_DATABASE`, ...) and no `/opt/bitnami/scripts/*`. Anything that + `exec`s into the pod and relies on those needs rewriting against + `POSTGRES_*`, or better, pointed at the Service over the network, which is + unaffected. +* **`kubectl exec ... pg_dumpall` without credentials still works, but for a + different reason.** The official image's `initdb` leaves `local` connections + on `trust`, and the container runs as the `postgres` OS user, so a dump over + the unix socket needs no password. This depends on the `/var/run/postgresql` + mount being present; if you override `postgresql.extraVolumeMounts` in a way + that removes it, socket connections, and the server itself, stop working. + ## Upgrading to 2.0.0 **Breaking Changes for OIDC Users** @@ -75,7 +546,7 @@ deployment.apps/nebraska scaled 2. Backup PostgreSQL data: ``` -$ kubectl exec -ti pod/nebraska-postgresql-0 -- pg_dumpall > backup.sql +$ kubectl exec -ti pod/nebraska-postgresql-0, pg_dumpall > backup.sql ``` 3. Scale down Nebraska statefulset: @@ -86,20 +557,33 @@ statefulset.apps/nebraska-postgresql scaled 4. Backup and remove the data from the bound volume (depending on the storage class) -3. Upgrade PostgreSQL version, e.g: +5. Upgrade PostgreSQL version, e.g: ```diff -- image: docker.io/bitnami/postgresql:13.8.0-debian-11-r18 -+ image: docker.io/bitnamilegacy/postgresql:17.5.0 +- tag: 17-bookworm ++ tag: 18-bookworm ``` + **The mount path must move with the major version.** PostgreSQL 18 relocates + both `PGDATA` and the image's declared `VOLUME`: + + | major | set `dataMountPath` | set `dataSubdir` | + |-------|---------------------|------------------| + | 17 | `/var/lib/postgresql/data` | `pgdata` | + | 18 | `/var/lib/postgresql` | `18/docker` | + + Get this wrong and the failure is silent: mounting the PVC *above* the + image's `VOLUME` makes the runtime lay an empty volume over the top, so + everything already on your disk becomes invisible inside the container. The + chart refuses the combination rather than letting it happen, but only when + it can read the major version from the tag. -5. Apply the changes and scale up Nebraska statefulset to its original value +6. Apply the changes and scale up Nebraska statefulset to its original value -6. Inject the backup and assert that everything looks good in the database: +7. Inject the backup and assert that everything looks good in the database: ``` -$ kubectl exec -ti pod/nebraska-postgresql-0 -- psql < backup.sql +$ kubectl exec -ti pod/nebraska-postgresql-0, psql < backup.sql ``` -7. Scale up Nebraska deployment and assert that everything is back to normal +8. Scale up Nebraska deployment and assert that everything is back to normal ## Parameters @@ -141,7 +625,7 @@ $ kubectl exec -ti pod/nebraska-postgresql-0 -- psql < backup.sql | `ingress.hosts` | Hostname(s) for the Ingress resource | `["flatcar.example.com"]` | | `ingress.ingressClassName` | Ingress controller which implements the resource. This replaces the deprecated `kubernetes.io/ingress.class` annotation on K8s > 1.19 | `""` | | `ingress.tls` | Ingress TLS configuration | `[]` | -| `ingress.update.enabled` | Create a separate ingress for the `/v1/update` and `/flatcar` paths, with it's own annotations. | `false` | +| `ingress.update.enabled` | Create a separate ingress for the `/v1/update` and `/flatcar` paths, with its own annotations. | `false` | | `ingress.update.annotations` | Annotations for Ingress resource | `{}` | | `ingress.update.ingressClassName` | Ingress controller which implements the resource. This replaces the deprecated `kubernetes.io/ingress.class` annotation on K8s > 1.19 | `""` | | `resources` | CPU/Memory resource requests/limits | `{}` | @@ -166,11 +650,11 @@ $ kubectl exec -ti pod/nebraska-postgresql-0 -- psql < backup.sql | `config.hostFlatcarPackages.packagesPath` | Path where Flatcar packages files should be stored | `/mnt/packages` | | `config.hostFlatcarPackages.nebraskaURL` | Nebraska URL (`http://host:port`) | `nil` (defaults to first ingress host) | | `config.hostFlatcarPackages.persistence.enabled` | Enable persistence using PVC | `false` | -| `config.hostFlatcarPackages.persistence.labels | Additional labels to be applied to the PVC | | `nil` | -| `config.hostFlatcarPackages.persistence.annotations | Additional annotations to be applied to the PVC | | `nil` | -| `config.hostFlatcarPackages.persistence.storageClass` | PVC Storage Class for PostgreSQL volume | `nil` | -| `config.hostFlatcarPackages.persistence.accessModes` | PVC Access Mode for PostgreSQL volume | `["ReadWriteOnce"]` | -| `config.hostFlatcarPackages.persistence.size` | PVC Storage Request for PostgreSQL volume | `10Gi` | +| `config.hostFlatcarPackages.persistence.labels` | Additional labels to be applied to the PVC | `nil` | +| `config.hostFlatcarPackages.persistence.annotations` | Additional annotations to be applied to the PVC | `nil` | +| `config.hostFlatcarPackages.persistence.storageClass` | PVC Storage Class for the Flatcar packages volume | `nil` | +| `config.hostFlatcarPackages.persistence.accessModes` | PVC Access Mode for the Flatcar packages volume | `["ReadWriteOnce"]` | +| `config.hostFlatcarPackages.persistence.size` | PVC Storage Request for the Flatcar packages volume | `10Gi` | | `config.caFile` | Path to a PEM-encoded CA certificate file to trust for TLS verification (additive to system CAs, used for OIDC and syncer) | `nil` | | `config.auth.mode` | Authentication mode, available modes: `noop`, `github`, `oidc` | `noop` | | `config.auth.github.clientID` | GitHub client ID used for authentication | `nil` | @@ -193,27 +677,36 @@ $ kubectl exec -ti pod/nebraska-postgresql-0 -- psql < backup.sql | `config.auth.oidc.scopes` | comma-separated list of scopes to be used in OIDC | `nil` | | `config.auth.oidc.audience` | OIDC audience (required for Auth0, optional for others) | `nil` | | `config.auth.oidc.useUserInfo` | Use UserInfo endpoint for role extraction (for providers that don't include roles in access token) | `false` | -| `config.database.host` | The host name of the database server | `""` (use postgresql from Bitnami subchart) | -| `config.database.port` | The port number the database server is listening on | `5432` | +| `config.database.host` | The host name of the database server | `""` (use the PostgreSQL bundled with this chart) | +| `config.database.port` | The port number the database server is listening on | `""` (follows `postgresql.service.port` when bundled, else 5432) | | `config.database.sslMode` | The mode of the database connection | `disable` | | `config.database.dbname` | The database name | `{{ .Values.postgresql.auth.database }}` (evaluated as a template) | -| `config.database.username` | PostgreSQL user | `{{ .Values.postgresql.postgresqlUsername }}` (evaluated as a template) | +| `config.database.username` | PostgreSQL user | `{{ .Values.postgresql.auth.username }}` (evaluated as a template) | | `config.database.password` | PostgreSQL user password | `""` (evaluated as a template) | | `config.database.passwordExistingSecret.enabled` | Enables setting PostgreSQL user password via an existing secret | `true` | -| `config.database.passwordExistingSecret.name` | Name of the existing secret | `{{ .Release.Name }}-postgresql` (evaluated as a template) | +| `config.database.passwordExistingSecret.name` | Name of the existing secret | `{{ include "nebraska.postgresql.secretName" . }}` (follows `existingSecret`/name overrides) | | `config.database.passwordExistingSecret.key` | Key inside the existing secret containing the PostgreSQL user password | `postgres-password` | | `extraArgs` | Extra arguments to pass to Nebraska binary | `[]` | | `extraEnvVars` | Any extra environment variables you would like to pass on to the pod | `{ "TZ": "UTC" }` | | `extraEnv` | Any extra environment variables in the form of env spec to pass into the deployment pod | `[]` | -### Postgresql dependency +### Bundled PostgreSQL parameters | Parameter | Description | Default | |----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------|------------------------| -| `postgresql.enabled` | Enable Bitnami postgresql subchart and deploy database within this helm release | `true` | +| `postgresql.enabled` | Deploy the PostgreSQL StatefulSet bundled with this chart | `true` | | `postgresql.auth.database` | PostgreSQL database | `nebraska` | -| `postgresql.auth.postgresPassword` | PostgreSQL password of user "postgres" **Recommended to change it to something secure for security reasons.** | `changeIt` | -| `postgresql.image.tag` | PostgreSQL Image tag | `13.8.0-debian-11-r18` | +| `postgresql.auth.postgresPassword` | PostgreSQL password of user "postgres" | `""` (a random password is generated on first install) | +| `postgresql.image.repository` | PostgreSQL image repository | `postgres` | +| `postgresql.image.tag` | PostgreSQL Image tag | `17-bookworm` | +| `postgresql.image.pullSecrets` | Image pull secrets. Accepts the Bitnami string form (`[regcred]`) and the object form (`[{name: regcred}]`) | `[]` | +| `postgresql.auth.existingSecret` | Use an existing secret for the password instead of rendering one (evaluated as a template) | `""` | +| `postgresql.auth.secretKeys.adminPasswordKey` | Key inside the secret holding the password | `postgres-password` | +| `postgresql.dataMountPath` | Where the data volume is mounted | `/var/lib/postgresql/data` | +| `postgresql.dataSubdir` | Subdirectory of the mount used as `PGDATA` (must not be the mount root) | `pgdata` | +| `postgresql.podSecurityContext` | Pod security context; uid/gid 999 matches the default Debian image (use 70 for Alpine tags) | see `values.yaml` | +| `postgresql.containerSecurityContext` | Container security context; `readOnlyRootFilesystem` is on by default | see `values.yaml` | +| `postgresql.resources` | Resource requests/limits for the PostgreSQL container | `250m` / `256Mi` requests | | `postgresql.primary.persistence.enabled` | Enable persistence using PVC | `false` | | `postgresql.primary.persistence.storageClass` | PVC Storage Class for PostgreSQL volume | `nil` | | `postgresql.primary.persistence.accessModes` | PVC Access Mode for PostgreSQL volume | `["ReadWriteOnce"]` | @@ -221,4 +714,7 @@ $ kubectl exec -ti pod/nebraska-postgresql-0 -- psql < backup.sql | `postgresql.serviceAccount.create` | Enable creation of ServiceAccount for PostgreSQL pod | `true` | | `postgresql.serviceAccount.automountServiceAccountToken` | Can be set to false if pods using this serviceAccount do not need to use K8s API | `false` | -... for more options see https://github.com/bitnami/charts/tree/master/bitnami/postgresql +This is a deliberately minimal, single-replica PostgreSQL meant to make `helm install` work out of +the box. It does no backups, no failover and no automated major-version upgrades. For production, +set `postgresql.enabled: false` and point `config.database.*` at a database you operate, or at an +operator such as [CloudNativePG](https://cloudnative-pg.io/). diff --git a/charts/nebraska/templates/NOTES.txt b/charts/nebraska/templates/NOTES.txt index b8dfd8a9e..3dc9c8caa 100644 --- a/charts/nebraska/templates/NOTES.txt +++ b/charts/nebraska/templates/NOTES.txt @@ -1,4 +1,50 @@ -1. Get the application URL by running these commands: +{{- if and .Values.postgresql.enabled .Values.postgresql.primary.persistence.enabled }} +NOTE ON THE BUNDLED POSTGRESQL + + This release stores its database in the volume claim + data-{{ include "nebraska.postgresql.fullname" . }}-0, at the path + {{ .Values.postgresql.dataMountPath }}/{{ .Values.postgresql.dataSubdir }}. + + If you have just upgraded from chart 2.0.0, that path is NOT where the Bitnami + subchart kept its data. Check that your data is really there, first that the + schema exists at all, then that it has rows: + + kubectl exec -n {{ .Release.Namespace }} {{ include "nebraska.postgresql.fullname" . }}-0, \ + psql -U {{ .Values.postgresql.auth.username }} -d {{ .Values.postgresql.auth.database }} \ + -tAc "select to_regclass('public.application')" + # empty output = no schema at all = PostgreSQL started a FRESH cluster + + kubectl exec -n {{ .Release.Namespace }} {{ include "nebraska.postgresql.fullname" . }}-0, \ + psql -U {{ .Values.postgresql.auth.username }} -d {{ .Values.postgresql.auth.database }} \ + -tAc "select count(*) from application" + # run only if the first command printed a name + + Empty output from the first command, or 0 from the second on an install that + had data, means your old cluster is still sitting unused in the volume's + data/ directory. + + kubectl rollout undo is NOT the fix. Run: helm rollback {{ .Release.Name }} + Do this BEFORE deleting any PVC or PersistentVolume. It is recoverable. + + Note the Nebraska pod IS restarted by this upgrade (chart 3.0.0 carries a + pod-template annotation for exactly this reason), so it re-runs its schema + migrations against whatever database it now points at. Its liveness probe does + not touch the database either way, so do not treat a Ready pod as confirmation + , run the two commands above. +{{ end -}} +{{- if and .Values.postgresql.enabled (eq (.Values.postgresql.auth.postgresPassword | toString) "changeIt") (not .Values.postgresql.auth.existingSecret) }} +WARNING: the PostgreSQL superuser password is set to "changeIt", the fixed + default that chart 2.0.0 shipped. 3.0.0 generates a random one instead, + so this almost certainly came from a carried-over values file. + + Anything on the pod network that knows this default has full control of the + database, including command execution inside its container. There is no + NetworkPolicy restricting access to port 5432. + + Set postgresql.auth.postgresPassword, or point postgresql.auth.existingSecret + at a secret you manage. +{{ end -}} +Get the application URL by running these commands: {{- if .Values.ingress.enabled }} {{- range .Values.ingress.hosts }} http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }} @@ -9,7 +55,7 @@ echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "nebraska.fullname" . }}' + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "nebraska.fullname" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "nebraska.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} diff --git a/charts/nebraska/templates/_helpers.tpl b/charts/nebraska/templates/_helpers.tpl index c94f9af07..2a8d3e728 100644 --- a/charts/nebraska/templates/_helpers.tpl +++ b/charts/nebraska/templates/_helpers.tpl @@ -62,13 +62,187 @@ Create the name of the service account to use {{- end }} {{/* -Create a default fully qualified postgresql name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +Fully qualified name for the bundled PostgreSQL objects. + +[PARITY] This reproduces the Bitnami subchart's `common.names.fullname` semantics exactly, +including honouring fullnameOverride and the `contains` short-circuit that skips +the suffix when the release name already contains the component name. Those are +not stylistic details: any divergence renames the StatefulSet, which orphans the +PVC and hands the user a new empty database. A release named `nebraska` with +nameOverride `nebraska` produced `nebraska` before and would produce +`nebraska-nebraska` under a naive implementation. */}} {{- define "nebraska.postgresql.fullname" -}} +{{- if .Values.postgresql.fullnameOverride -}} +{{- .Values.postgresql.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} {{- $name := default "postgresql" .Values.postgresql.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Standard metadata for the bundled PostgreSQL objects. + +Factored out because the same labels/annotations stanza appeared on each of +them. Used by four of the five: the Secret inlines its own copy because it also +carries `helm.sh/resource-policy: keep`, and merging one fixed annotation into +this helper would cost more indirection than the six duplicated lines. Emits the +`annotations:` key only when there is something to put under it, so it stays +valid when extraAnnotations is empty. +*/}} +{{- define "nebraska.postgresql.metadata" -}} +labels: + {{- include "nebraska.postgresql.labels" . | nindent 2 }} + {{- with .Values.extraLabels }} + {{- toYaml . | nindent 2 }} + {{- end }} +{{- with .Values.extraAnnotations }} +annotations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + +{{/* +The readiness/liveness/startup check. One definition, three call sites. +*/}} +{{- define "nebraska.postgresql.probeCommand" -}} +exec: + command: + - /bin/sh + - -c + - exec pg_isready -U {{ .Values.postgresql.auth.username | quote }} -d {{ printf "dbname=%s" .Values.postgresql.auth.database | quote }} -h 127.0.0.1 -p {{ int .Values.postgresql.service.port }} +{{- end -}} + +{{/* +[PARITY] ServiceAccount used by the PostgreSQL pod. Honours an explicit name, as the +Bitnami subchart did, so an existing install pinning one keeps working. +*/}} +{{- define "nebraska.postgresql.serviceAccountName" -}} +{{- if .Values.postgresql.serviceAccount.create -}} +{{- default (include "nebraska.postgresql.fullname" .) .Values.postgresql.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.postgresql.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* +Name of the headless Service backing the StatefulSet. + +[EXTRA, not required by the migration] A pre-existing bug that the Bitnami +subchart had too, handled with exact parity wherever a working install can exist. + +The subchart computed `printf "%s-hl" fullname | trunc 63 | trimSuffix "-"`. +At release-name length 50 the fullname is 61 chars, so that yields +`-h`, distinct from the main Service, a working install, and an +immutable `spec.serviceName` that this chart has to reproduce exactly. So the +Bitnami expression is used verbatim whenever its result differs from the main +Service name. + +At lengths 51-52 the truncation drops the whole "-hl" (and trimSuffix drops +the dangling "-"), so the headless name COLLIDES with the main Service name and +the apiserver rejects the release: no working install can exist to stay +compatible with there. Only for that case the base is truncated to 60 before +appending, which is guaranteed distinct and <=63. (Appending without truncating +would give 66 characters, equally rejected.) +*/}} +{{- define "nebraska.postgresql.headlessName" -}} +{{- $fullname := include "nebraska.postgresql.fullname" . -}} +{{- $bitnami := printf "%s-hl" $fullname | trunc 63 | trimSuffix "-" -}} +{{- if ne $bitnami $fullname -}} +{{- $bitnami -}} +{{- else -}} +{{- printf "%s-hl" ($fullname | trunc 60 | trimSuffix "-") -}} +{{- end -}} +{{- end -}} + +{{/* +Resolve the StorageClass for the data volume. + +[PARITY] Honours global.storageClass like the subchart did, and reproduces its "-" +sentinel, which means "render an empty storageClassName" (bind to a pre-created +PV / disable dynamic provisioning) rather than "use a StorageClass literally +named -". Emits nothing when unset, so the cluster default applies. +*/}} +{{- define "nebraska.postgresql.storageClass" -}} +{{- $sc := .Values.postgresql.primary.persistence.storageClass | default (.Values.global | default dict).storageClass -}} +{{- if $sc -}} +{{- if eq $sc "-" -}} +storageClassName: "" +{{- else -}} +storageClassName: {{ $sc | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Selector labels for the bundled PostgreSQL StatefulSet. + +[MIGRATION] These are the same labels the Bitnami postgresql 11.9.1 subchart +used. A StatefulSet spec.selector cannot be changed after it is created, so +keeping them the same lets `helm upgrade` patch the existing StatefulSet instead +of failing. Do not change them without a major chart bump. + +Why they are kept as they are: + +If we renamed them to match the rest of the chart (for example +`app.kubernetes.io/name: nebraska` with `component: postgresql`) the labels +would read better, but every existing install would fail to upgrade. The user +would have to run `kubectl delete statefulset --cascade=orphan` by hand first. + +3.0.0 is a breaking release anyway, so this was the cheapest moment to rename +them. We decided not to. Upgrading in place without manual steps is worth more +than nicer labels. The price is that the chart keeps a selector that looks +wrong, `name: postgresql` in a chart called nebraska, and we cannot fix that +until the next major version. + +If this is revisited later, the change is small. It only affects this +StatefulSet selector and the labels built from it. It does not affect the +Nebraska Deployment, the Secret, or the data. +*/}} +{{- define "nebraska.postgresql.selectorLabels" -}} +{{- /* The name label follows nameOverride (NOT fullnameOverride), because the + subchart's `common.names.name` is `default .Chart.Name .Values.nameOverride` + and the StatefulSet's immutable selector embeds it. A 2.0.0 install with + postgresql.nameOverride=pg has `name: pg` in its selector; hardcoding + "postgresql" here would change an immutable field and get the upgrade + REJECTED by the apiserver, verified against the vendored subchart. */ -}} +app.kubernetes.io/name: {{ default "postgresql" .Values.postgresql.nameOverride }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: primary +{{- end -}} + +{{/* +Common labels for the bundled PostgreSQL objects. +*/}} +{{- define "nebraska.postgresql.labels" -}} +helm.sh/chart: {{ include "nebraska.chart" . }} +{{ include "nebraska.postgresql.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: {{ include "nebraska.name" . }} +{{- end -}} + +{{/* +Name of the secret holding the PostgreSQL superuser password. +*/}} +{{- define "nebraska.postgresql.secretName" -}} +{{- if .Values.postgresql.auth.existingSecret -}} +{{- $name := tpl .Values.postgresql.auth.existingSecret . -}} +{{- /* Naming the chart's own secret suppresses its template while leaving both + workloads referencing it, so Helm deletes it on upgrade and everything + fails with CreateContainerConfigError, with the password gone. */ -}} +{{- if eq $name (include "nebraska.postgresql.fullname" .) -}} +{{- fail (printf "postgresql.auth.existingSecret must not name the secret this chart manages (%s): Helm would stop rendering it and then delete it, taking the password with it. Either drop existingSecret and set postgresql.auth.postgresPassword, or point it at a separately-managed secret." $name) -}} +{{- end -}} +{{- $name -}} +{{- else -}} +{{- include "nebraska.postgresql.fullname" . -}} +{{- end -}} +{{- end -}} {{/* Return the appropriate apiVersion for ingress @@ -88,8 +262,9 @@ http{{ if $.Values.ingress.tls }}s{{ end }} {{- end -}} {{/* -Return the proper image name -This allows usage of global overrides for the image registry in a similar way the postgresql subchart does. +Return the proper image name. +This honours global overrides for the image registry the same way the former +postgresql subchart did. */}} {{- define "nebraska.image" -}} {{- $registryName := .imageRoot.registry -}} @@ -105,5 +280,11 @@ This allows usage of global overrides for the image registry in a similar way th {{- $registryName = .global.imageRegistry -}} {{- end -}} {{- end -}} +{{- /* A digest is content-addressed, so it pins the image regardless of what the + tag later points at. When set it replaces the tag entirely. */ -}} +{{- if .imageRoot.digest -}} +{{- printf "%s/%s@%s" $registryName $repositoryName (.imageRoot.digest | toString) -}} +{{- else -}} {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} {{- end -}} +{{- end -}} diff --git a/charts/nebraska/templates/_validate.tpl b/charts/nebraska/templates/_validate.tpl new file mode 100644 index 000000000..ae948dfd9 --- /dev/null +++ b/charts/nebraska/templates/_validate.tpl @@ -0,0 +1,407 @@ +{{/* +Refuse to upgrade a persistent install onto the new data directory layout +without an explicit acknowledgement. + +This is the most important guard in the chart. The Bitnami subchart mounted the +PVC at /bitnami/postgresql with PGDATA=/bitnami/postgresql/data; this chart +mounts the same PVC at /var/lib/postgresql/data with PGDATA=.../data/pgdata. + +Every field the Kubernetes StatefulSet controller treats as immutable -- +spec.selector, spec.serviceName, spec.volumeClaimTemplates, is deliberately +exactly what the subchart emitted, so `helm upgrade` is ACCEPTED and +returns 0. The pod restarts, finds no PG_VERSION at the new PGDATA, runs initdb +into a fresh sibling directory, and comes up as an empty cluster. Nebraska's +pod spec is unchanged by the upgrade, so it is NOT restarted and keeps serving +from stale connections; it recreates its schema on its next restart. Every +application, group, channel and rollout disappears from the user's point of +view while the old cluster sits untouched next to it. +No error is produced anywhere. + +Prose in the README does not defend against this: nobody reads a README during +a Renovate bump. So the render fails until the operator says they have dealt +with the data. + +The old cluster is still on the volume, so an accidental upgrade is recoverable +with `helm rollback`. That is worth knowing, so the message says it. + +This file only exists for the migration. Nothing in it prints anything when the +values are fine, so it can be removed a release or two after 3.0.0, once nobody +upgrades straight from 2.0.0 any more. + +If you tidy this file, be careful with the scan loops in validateUnknownValues. +They look repetitive and easy to merge, but two bugs were found in them during +review, and one earlier attempt to simplify them broke the chart. Re-run the +guard checks after any change here. +*/}} +{{- define "nebraska.postgresql.validateDataDirMigration" -}} +{{- $pg := .Values.postgresql | default dict -}} +{{- /* Only an UPGRADE can destroy data this way. A fresh install has no prior + cluster on the volume, so failing there would be pure friction for every + new user who wants persistence. Note this means template-rendering + workflows (Argo/Flux, `helm template`) never see the gate, because + .Release.IsUpgrade is false there, those users get the README and + NOTES.txt instead. */ -}} +{{- /* Compare as a string: `--set-string ...=false` passes the STRING "false", + which is truthy, so a truthiness check reads it as an acknowledgement and + the gate stays down. Only an actual true (bool or string) acknowledges. */ -}} +{{- if and .Release.IsUpgrade $pg.enabled ((($pg.primary | default dict).persistence | default dict).enabled) -}} +{{- if ne ($pg.acknowledgeDataDirMigration | toString) "true" -}} +{{- fail "\n\nSTOP. This upgrade would silently discard your database.\n\nChart 3.0.0 replaced the Bitnami postgresql subchart with the official postgres\nimage, which stores data at a different path inside the same volume:\n\n chart 2.0.0 (Bitnami): PVC mounted at /bitnami/postgresql PGDATA=/bitnami/postgresql/data\n chart 3.0.0 (official): PVC mounted at /var/lib/postgresql/data PGDATA=/var/lib/postgresql/data/pgdata\n\nNothing in Kubernetes rejects this change, so `helm upgrade` would SUCCEED and\nPostgreSQL would initialise a brand-new empty database alongside your existing\none. Nebraska would come up looking healthy with no applications, groups or\nrollouts, and no error would be reported.\n\nYou have persistence enabled, so you must choose:\n\n 1. Migrate the data (dump/restore). Follow \"Upgrading to 3.0.0\" in the chart\n README, then re-run with:\n --set postgresql.acknowledgeDataDirMigration=true\n\n 2. Deliberately start from an empty database (fine for dev/test). Same flag:\n --set postgresql.acknowledgeDataDirMigration=true\n\n 3. Stay on chart 2.0.0 for now.\n\nIf you already ran this upgrade by accident: your old cluster is still present\non the volume, untouched, in the `data/` directory. Run `helm rollback` NOW,\nbefore deleting any PVC, and it will come back.\n" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Report values that this chart does not honour. + +DENY-UNKNOWN, not allow-known. The first version of this listed the Bitnami keys +someone remembered, and a review of it found that it missed about 55 of them +-- including `primary.resources` (memory limits silently vanish), +`primary.podSecurityContext` (a user's fsGroup silently replaced), +`primary.persistence.existingClaim` (silently ignored, so a fresh EMPTY volume is +provisioned) and the whole `global.*` tree. An allowlist can only ever catch the +keys its author thought of, which is the wrong failure mode for a data-bearing +chart. So: enumerate the keys this chart READS, and report everything else. + +DEFAULT-AWARE. A very common pattern is to vendor the upstream subchart's +values.yaml wholesale and edit a few lines. Such a file carries dozens of keys at +their Bitnami defaults, `architecture: standalone`, `metrics.enabled: false`, +`tls.enabled: false`. Failing on those would tell users that features they never +turned on have been removed, which is noise, and would make a routine upgrade +impossible for exactly the people we least want to break. So an inert value +(a feature block that is disabled, or a setting already at the Bitnami default) +is ignored; only values that would actually have changed behaviour are reported. +*/}} + +{{/* True when a removed value would not have done anything anyway. + + Recursive, because a vendored Bitnami values.yaml carries whole nested + blocks sitting at their defaults. `metrics:` is not just `enabled: false` -- + it is fifty lines of image tags, probe timings and resource stanzas hanging + off it. Reporting every one of those would make the upgrade unusable for + precisely the people the README promises it will work for. + + A value is inert when it is empty, false, an empty collection, a feature + block that is switched off, a map whose every member is itself inert, or + a setting sitting at its Bitnami default (e.g. architecture: standalone). */}} +{{- define "nebraska.postgresql.isInertValue" -}} +{{- $v := .value -}} +{{- $k := .key -}} +{{- if kindIs "invalid" $v -}}inert +{{- else if kindIs "bool" $v -}} + {{- if not $v -}}inert{{- end -}} +{{- else if kindIs "string" $v -}} + {{- if eq $v "" -}}inert + {{- else if and (eq $k "architecture") (eq $v "standalone") -}}inert{{- end -}} +{{- else if kindIs "slice" $v -}} + {{- if not $v -}}inert{{- end -}} +{{- else if kindIs "map" $v -}} + {{- if not $v -}}inert + {{- else if and (hasKey $v "enabled") (not $v.enabled) -}}inert + {{- else if and (hasKey $v "create") (not $v.create) -}}inert + {{- else -}} + {{- $allInert := true -}} + {{- range $ck, $cv := $v -}} + {{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $ck "value" $cv)) -}} + {{- $allInert = false -}} + {{- end -}} + {{- end -}} + {{- if $allInert -}}inert{{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "nebraska.postgresql.validateUnknownValues" -}} +{{- $pg := .Values.postgresql | default dict -}} +{{- $found := list -}} + +{{/* Keys this chart actually reads. Anything else is reported. */}} +{{- $known := list + "enabled" "acknowledgeDataDirMigration" "nameOverride" "fullnameOverride" + "extraPodSpec" + "auth" "image" "service" "dataMountPath" "dataSubdir" "primary" + "serviceAccount" "podSecurityContext" "containerSecurityContext" + "terminationGracePeriodSeconds" + "resources" + "extraEnv" "extraVolumes" "extraVolumeMounts" + "args" "shmSizeLimit" "startupProbe" +-}} + +{{/* Specific guidance where a generic message would not be enough. */}} +{{- $guide := dict + "architecture" "streaming standbys are not provided by this chart. Note Nebraska cannot use a read-only standby anyway. Every Omaha check-in writes, and it opens a single DSN. If you are after the distributed topology in RFC #1375, that uses one-way LOGICAL replication, which this chart can do: set postgresql.args to include -c wal_level=logical. For managed HA, use an operator such as CloudNativePG with postgresql.enabled=false." + "replication" "streaming replication is not provided by this chart. Logical replication is reachable via postgresql.args (-c wal_level=logical); for managed HA use an operator with postgresql.enabled=false." + "readReplicas" "read replicas are not provided by this chart, and Nebraska opens a single DSN so it has no read/write split to use them." + "metrics" "the postgres-exporter sidecar, ServiceMonitor and PrometheusRule are gone. Run postgres-exporter as its own Deployment against the PostgreSQL Service, and supply it plus any ServiceMonitor through the top-level extraObjects." + "tls" "in-chart TLS termination is gone. Set server options via postgresql.args (-c ssl=on -c ssl_cert_file=...) with the cert supplied through postgresql.extraVolumes, or terminate at a proxy/mesh." + "ldap" "LDAP auth is gone. It was never used by Nebraska." + "audit" "pgAuditLog/pgAuditLogCatalog need the pgaudit extension, which the official image does not ship. The other audit settings (logConnections, logDisconnections, logHostname, logLinePrefix, logTimezone, clientMinMessages) are plain PostgreSQL settings, set them via postgresql.args, e.g. -c log_connections=on." + "postgresqlSharedPreloadLibraries" "set this via postgresql.args (-c shared_preload_libraries=...). Note the official image does not ship pgaudit." + "postgresqlDataDir" "renamed. Use postgresql.dataMountPath plus postgresql.dataSubdir; PGDATA must be a strict subdirectory of the mount." + "volumePermissions" "podSecurityContext.fsGroup handles ownership on CSI drivers that honour it. On storage that ignores fsGroup (some NFS), reproduce the chown with postgresql.extraPodSpec.initContainers, see the worked example in values.yaml." + "networkPolicy" "NetworkPolicy is not rendered by this chart. Supply your own through the top-level extraObjects." + "rbac" "no Role/RoleBinding is needed; the pod does not talk to the API server." + "psp" "PodSecurityPolicy was removed from Kubernetes in 1.25." + "shmVolume" "/dev/shm is always mounted as a Memory-backed emptyDir. Use postgresql.shmSizeLimit to bound it." + "containerPorts" "renamed. Use postgresql.service.port." + "extraDeploy" "renamed. Use the top-level extraObjects." + "commonLabels" "renamed. Use the top-level extraLabels." + "commonAnnotations" "renamed. Use the top-level extraAnnotations." + "clusterDomain" "not used; the chart addresses the database by Service name." + "diagnosticMode" "not supported. Use postgresql.args to change the server command line." +-}} + +{{- range $k, $v := $pg -}} +{{- if not (has $k $known) -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $help := index $guide $k | default "not read by this chart; check charts/nebraska/values.yaml for the current key." -}} +{{- $found = append $found (printf "postgresql.%s: %s" $k $help) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- $knownAuth := list "username" "database" "postgresPassword" "existingSecret" "secretKeys" -}} +{{- $guideAuth := dict + "password" "the official image has a single superuser password. Use postgresql.auth.postgresPassword (secret key postgres-password)." + "enablePostgresUser" "POSTGRES_USER is always the superuser initdb creates; there is no separate postgres role to toggle." + "replicationUsername" "replication is not provided by this chart." + "replicationPassword" "replication is not provided by this chart." + "usePasswordFiles" "not supported; the password is injected with secretKeyRef." +-}} +{{- range $k, $v := ($pg.auth | default dict) -}} +{{- if not (has $k $knownAuth) -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $found = append $found (printf "postgresql.auth.%s: %s" $k (index $guideAuth $k | default "not read by this chart.")) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Recurse one level into auth.secretKeys: only adminPasswordKey is read, and + Bitnami's siblings (userPasswordKey, replicationPasswordKey) were passing + silently, exactly the class of gap the deny-unknown design exists to + close, missed because the recursion stopped at auth.* */}} +{{- range $k, $v := (($pg.auth | default dict).secretKeys | default dict) -}} +{{- if ne $k "adminPasswordKey" -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $found = append $found (printf "postgresql.auth.secretKeys.%s: not read by this chart. The official image has a single superuser, so only adminPasswordKey applies." $k) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Recurse into serviceAccount: only create/name/automountServiceAccountToken + are read. annotations is the one to watch, because it passes the top level check + serviceAccount is known, then is silently discarded. */}} +{{- $knownSA := list "create" "name" "automountServiceAccountToken" -}} +{{- $guideSA := dict + "annotations" "not read for the PostgreSQL ServiceAccount. Use the top-level extraAnnotations, which reach every object." +-}} +{{- range $k, $v := (($pg.serviceAccount | default dict)) -}} +{{- if not (has $k $knownSA) -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $found = append $found (printf "postgresql.serviceAccount.%s: %s" $k (index $guideSA $k | default "not read by this chart.")) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* extraPodSpec is merged at pod-spec level; `containers` or `volumes` there + emit a SECOND key alongside the chart's own, producing an invalid pod spec + (the values.yaml example says as much in prose, prose is not a guard). */}} +{{- if $pg.enabled -}} +{{- range $k, $v := ($pg.extraPodSpec | default dict) -}} +{{- if has $k (list "containers" "volumes") -}} +{{- $found = append $found (printf "postgresql.extraPodSpec.%s: duplicates a key the chart renders itself, which makes the pod spec invalid. Set extraEnv/extraVolumes/extraVolumeMounts (first-class values) or use initContainers via extraPodSpec." $k) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Most `primary.*` keys just moved up a level, so derive that message from + the known-keys list rather than writing a near-identical table entry for + each by hand. Only keys whose replacement is NOT a simple rename need a + bespoke message below. `extraEnvVars` is the one rename that changed name + as well as level, so it is listed explicitly. */}} +{{- $guidePrimary := dict + "configuration" "custom postgresql.conf is not rendered. Set server options with postgresql.args, e.g. -c max_connections=200." + "extendedConfiguration" "set server options with postgresql.args." + "existingConfigmap" "set server options with postgresql.args, or mount your own file and point at it with postgresql.args." + "existingExtendedConfigmap" "set server options with postgresql.args." + "pgHbaConfiguration" "custom pg_hba.conf is not rendered. The official image's initdb writes pg_hba.conf inside PGDATA." + "initdb" "mount a ConfigMap at /docker-entrypoint-initdb.d with postgresql.extraVolumes and postgresql.extraVolumeMounts. Note it only runs on a first-time init of an empty data directory." + "standby" "standby/streaming replication is not provided by this chart." + "extraEnvVars" "moved to postgresql.extraEnv." + "extraEnvVarsCM" "not supported; the official image reads only a few POSTGRES_* variables, and only on first init. Use postgresql.extraEnv." + "extraEnvVarsSecret" "not supported; use postgresql.extraEnv with a secretKeyRef." + "podAntiAffinityPreset" "meaningless for a single replica; there is no second pod to schedule away from." + "updateStrategy" "fixed to RollingUpdate; with a single replica there is nothing else to choose." + "priorityClassName" "set it through postgresql.extraPodSpec." + "schedulerName" "set it through postgresql.extraPodSpec." + "hostAliases" "set it through postgresql.extraPodSpec." + "topologySpreadConstraints" "set it through postgresql.extraPodSpec." + "command" "not supported. Use postgresql.args to pass arguments to postgres." + "service" "only the port is configurable, as postgresql.service.port." + "resources" "moved to postgresql.resources. Left here your CPU/memory limits would be silently dropped." + "podSecurityContext" "moved to postgresql.podSecurityContext. Left here your runAsUser/fsGroup would be silently dropped, which matters, because the image's uid changed." + "livenessProbe" "probes are fixed by this chart. postgresql.startupProbe tunes the first-start budget." + "readinessProbe" "probes are fixed by this chart. postgresql.startupProbe tunes the first-start budget." + "lifecycleHooks" "not supported; the chart sets a preStop hook for clean shutdown." + "sidecars" "not supported. A metrics exporter or backup agent does not need to share the pod, run it as its own Deployment against the Service. If you need one badly enough, use postgresql.enabled=false and a real database." + "initContainers" "set them through postgresql.extraPodSpec.initContainers." + "nodeSelector" "set it through postgresql.extraPodSpec." + "tolerations" "set them through postgresql.extraPodSpec." + "affinity" "set it through postgresql.extraPodSpec." + "podLabels" "not offered on the PostgreSQL pod template; the top-level extraLabels apply to object metadata only and cannot be used in pod selectors. For NetworkPolicy, match the pod's existing labels through extraObjects." + "podAnnotations" "not offered on the PostgreSQL pod template. The top-level extraAnnotations apply to object metadata only." +-}} +{{- range $k, $v := ($pg.primary | default dict) -}} +{{- if ne $k "persistence" -}} +{{- if has $k (list "livenessProbe" "readinessProbe") -}} +{{- /* The subchart defaulted these to enabled: true, so a deliberate + `enabled: false` is a REAL change, not an inert default, isInertValue + would swallow it and silently turn the probe ON again. Report any + non-empty setting; only a completely empty value means nothing. */ -}} +{{- if $v -}} +{{- $found = append $found (printf "postgresql.primary.%s: probes are fixed by this chart. postgresql.startupProbe tunes the first-start budget." $k) -}} +{{- end -}} +{{- else if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $help := index $guidePrimary $k -}} +{{- if not $help -}} +{{- $help = ternary (printf "moved to postgresql.%s." $k) "not read by this chart." (has $k $known) -}} +{{- end -}} +{{- $found = append $found (printf "postgresql.primary.%s: %s" $k $help) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- $knownPersist := list "enabled" "storageClass" "accessModes" "size" "labels" "annotations" -}} +{{- $guidePersist := dict + "existingClaim" "not supported. The chart always uses a volumeClaimTemplate named 'data', so leaving this set would provision a NEW, EMPTY volume and leave your claim untouched." + "mountPath" "renamed. Use postgresql.dataMountPath." + "subPath" "not supported. Use postgresql.dataSubdir, which is the PGDATA subdirectory inside the volume." +-}} +{{- range $k, $v := (($pg.primary | default dict).persistence | default dict) -}} +{{- if not (has $k $knownPersist) -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $found = append $found (printf "postgresql.primary.persistence.%s: %s" $k (index $guidePersist $k | default "not read by this chart.")) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* The subchart honoured global.*; nothing here does except global.imageRegistry + and global.storageClass. global.postgresql.auth.* is Bitnami's own documented + way to set the password, so silently ignoring it would rewrite a live Secret + to the chart default while the database keeps the old password. */}} +{{- $global := .Values.global | default dict -}} +{{- if $global.postgresql -}} +{{- $found = append $found "global.postgresql.*: not read by this chart. Use postgresql.auth.* and postgresql.image.* instead." -}} +{{- end -}} +{{- range $k, $v := $global -}} +{{- if not (has $k (list "imageRegistry" "storageClass" "postgresql")) -}} +{{- if not (include "nebraska.postgresql.isInertValue" (dict "key" $k "value" $v)) -}} +{{- $found = append $found (printf "global.%s: not read by this chart; only global.imageRegistry and global.storageClass are honoured." $k) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- if $found -}} +{{- fail (printf "\n\nThese values are not read by this chart and would have been silently ignored:\n\n - %s\n\nChart 3.0.0 replaced the Bitnami postgresql subchart with an in-chart StatefulSet\nrunning the official postgres image. See \"Upgrading to 3.0.0\" in the chart README.\nValues left at their Bitnami defaults are not reported, so everything listed above\nwould really have changed behaviour.\n\nIf you need something this chart does not provide, set postgresql.enabled=false and\npoint config.database.* at a database you operate.\n" (join "\n - " $found)) -}} +{{- end -}} +{{- end -}} + +{{/* +Refuse a data mount that sits above the image's declared VOLUME. + +The postgres image declares a VOLUME. If the PVC is mounted at an ANCESTOR of +that path, the container runtime mounts an empty volume over the top and +everything the PVC holds underneath becomes invisible inside the container. The +pod starts, initdb runs into what looks like empty space, and the real data is +still on the PV where nobody can see it. + +docker-library documents this for <=17 ("mount at /var/lib/postgresql/data and +NOT at /var/lib/postgresql, or data WILL NOT PERSIST"). It is widely assumed to +be a Docker-only quirk that Kubernetes ignores. It is not, this was observed +live on kind, with a Bitnami data directory present on the PV and unreadable +from inside the pod. + +The VOLUME moved in 18 (/var/lib/postgresql/data -> /var/lib/postgresql), so the +correct mount point depends on the major version. The tag is parsed to pick the +right one; an unrecognisable tag is left alone rather than guessed at. + +Why this guard is here at all, since most people never touch dataMountPath: +it is a value this chart exposes, and the README tells people to change it when +they move to PostgreSQL 18, because the VOLUME moved in 18. So a user can reach +this problem by following our own instructions, not only by making a typo. The +failure is silent and complete: the pod starts, looks healthy, and the data is +invisible. That is worth a guard even if it is rarely hit. + +It is best effort by design. It is the only place the chart reads the image tag +to decide something, and an unknown tag is left alone instead of guessed. +*/}} +{{- define "nebraska.postgresql.validateMountPath" -}} +{{- $pg := .Values.postgresql | default dict -}} +{{- if $pg.enabled -}} +{{- $mount := $pg.dataMountPath | default "" | toString | trimSuffix "/" -}} +{{- $sub := $pg.dataSubdir | default "" | toString -}} +{{- /* PGDATA is mount/subdir, CONCATENATED AS TEXT and resolved by the kernel: + `../../../../tmp/pgdata` lands in the ephemeral /tmp emptyDir with the + pod looking healthy. Reject anything that can escape the volume; allow + clean nested values like `18/docker`. */ -}} +{{- if or (not $sub) (hasPrefix "/" $sub) (eq $sub ".") -}} +{{- fail (printf "\n\npostgresql.dataSubdir %q is not usable: it must be a non-empty relative\nsubdirectory of the volume mount (e.g. pgdata, or 18/docker for PostgreSQL 18).\n" $sub) -}} +{{- end -}} +{{- range $seg := (splitList "/" $sub) -}} +{{- if eq $seg ".." -}} +{{- fail (printf "\n\npostgresql.dataSubdir %q contains '..': PGDATA is computed as\ndataMountPath/dataSubdir and resolved by the kernel, so this escapes the data\nvolume (e.g. into the ephemeral /tmp emptyDir) while the pod looks healthy.\nUse a plain relative subdirectory like pgdata or 18/docker.\n" $sub) -}} +{{- end -}} +{{- end -}} +{{- if or (not $mount) (not (hasPrefix "/" $mount)) -}} +{{- fail (printf "\n\npostgresql.dataMountPath %q is not usable: it must be an absolute path\n(e.g. /var/lib/postgresql/data).\n" $mount) -}} +{{- end -}} +{{- range $seg := (splitList "/" $mount) -}} +{{- if eq $seg ".." -}} +{{- fail (printf "\n\npostgresql.dataMountPath %q contains '..'.\n" $mount) -}} +{{- end -}} +{{- end -}} +{{- $major := regexFind "^[0-9]+" ((($pg.image | default dict).tag | default "" | toString)) -}} +{{- if and $major $mount -}} +{{- $vol := ternary "/var/lib/postgresql" "/var/lib/postgresql/data" (ge (int $major) 18) -}} +{{- if hasPrefix (printf "%s/" $mount) $vol -}} +{{- fail (printf "\n\npostgresql.dataMountPath is %q, which is above the VOLUME the image declares (%q\nfor PostgreSQL %s).\n\nMounting the data volume above the image's VOLUME makes the runtime lay an empty\nvolume over the top: the pod starts, PostgreSQL initialises into what looks like\nempty space, and everything already on your PVC becomes invisible from inside the\ncontainer. It is still on the disk, and nothing will tell you.\n\nSet postgresql.dataMountPath to %q (and keep postgresql.dataSubdir as the\nsubdirectory inside it).\n" $mount $vol $major $vol) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Refuse a Bitnami-family image. + +The whole point of 3.0.0 is to stop shipping an image that never gets CVE fixes. +A values file carrying `postgresql.image.repository: bitnamilegacy/postgresql` +from 2.0.0 still renders, and the resulting pod fails in a way that looks like a +chart bug (the Bitnami entrypoint does not understand this chart's PGDATA layout) +rather than a stale value. Fail with an explanation instead. +*/}} +{{- define "nebraska.postgresql.validateImage" -}} +{{- $pg := .Values.postgresql | default dict -}} +{{- $img := $pg.image | default dict -}} +{{- /* Match the FULL EFFECTIVE reference, not just the repository. Checking + `repository` alone was bypassable by pushing the vendor name into the + registry (--set postgresql.image.registry=docker.io/bitnamilegacy), and + checking the local registry alone was bypassable by pushing it into + global.imageRegistry, which nebraska.image applies as an override. A + guard with a trivial bypass is worse than none, because it advertises + protection it does not provide. */ -}} +{{- $registry := ($img.registry | default "" | toString) -}} +{{- with .Values.global -}}{{- with .imageRegistry -}}{{- $registry = . -}}{{- end -}}{{- end -}} +{{- $ref := printf "%s/%s" $registry ($img.repository | default "" | toString) -}} +{{- /* Gated on enabled: with postgresql.enabled=false the image is never + pulled, so a stale override is inert and refusing would contradict the + README's "no action needed" for external-database users. */ -}} +{{- if and $pg.enabled (regexMatch "bitnami" (lower $ref)) -}} +{{- fail (printf "\n\nThe PostgreSQL image resolves to %q, which is a Bitnami-family image.\n\nChart 3.0.0 runs the official postgres image and configures it accordingly\n(PGDATA layout, uid, env var names). A Bitnami image will not start correctly\nhere, and bitnamilegacy/* is a frozen archive that receives no security updates\n-- which is the reason this chart stopped using it.\n\nRemove the postgresql.image override to use the chart default, or set\npostgresql.enabled=false and run your own database.\n" $ref) -}} +{{- end -}} +{{- /* An empty tag with no digest renders `postgres:`, unpullable, and the + appVersion fallback is deliberately disabled for this image (postgres + has no 3.x matching the chart's). Fail with a message instead. */ -}} +{{- if and $pg.enabled (not ($img.tag | default "" | toString)) (not ($img.digest | default "" | toString)) -}} +{{- fail "\n\npostgresql.image.tag is empty and no postgresql.image.digest is set, so the image\nwould render as 'postgres:' with no tag, unpullable. (The chart's appVersion\nfallback applies to the Nebraska image only; there is no postgres:3.0.0.)\nSet a tag (e.g. 17-bookworm) or a digest.\n" -}} +{{- end -}} +{{- end -}} diff --git a/charts/nebraska/templates/deployment.yaml b/charts/nebraska/templates/deployment.yaml index 65138a099..476daadc1 100644 --- a/charts/nebraska/templates/deployment.yaml +++ b/charts/nebraska/templates/deployment.yaml @@ -1,6 +1,9 @@ {{- $db := ( tpl .Values.config.database.dbname . ) }} {{- $host := .Values.config.database.host | default (include "nebraska.postgresql.fullname" .) }} -{{- $port := .Values.config.database.port | toString }} +{{- /* Port: an explicit config.database.port wins; otherwise follow the bundled + Service's port when postgresql.enabled (postgresql.service.port changes + the port PostgreSQL actually listens on) and 5432 for external hosts. */ -}} +{{- $port := .Values.config.database.port | default (ternary .Values.postgresql.service.port 5432 .Values.postgresql.enabled) | toString }} {{- $sslMode := .Values.config.database.sslMode | default "disable" }} {{- $user := ( tpl .Values.config.database.username . ) }} apiVersion: apps/v1 @@ -26,9 +29,28 @@ spec: {{- include "nebraska.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} + {{- /* [GUARD] The bundled database is REPLACED by this migration, and on + the default ephemeral install it comes back EMPTY. Nebraska applies + its schema migrations only at process start, and nothing else in + this Deployment changes between 2.0.0 and 3.0.0, so without a + pod-template change here Helm does not roll Nebraska, and it keeps + serving from a database with no tables while looking healthy (its + probes hit `/`, and /health returns 200 unconditionally). + + This annotation changes the pod template exactly once, so the + upgrade restarts Nebraska and it recreates its schema unattended. + Only bump the value if the bundled database is ever replaced again; + it is deliberately NOT derived from the image tag, because that + would also restart Nebraska (strategy: Recreate = downtime) on a + harmless PostgreSQL patch bump. */}} + {{- if or .Values.postgresql.enabled .Values.podAnnotations }} annotations: + {{- if .Values.postgresql.enabled }} + nebraska.flatcar.org/bundled-db-generation: "official-postgres-v1" + {{- end }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} labels: {{- include "nebraska.selectorLabels" . | nindent 8 }} @@ -148,8 +170,21 @@ spec: valueFrom: secretKeyRef: {{- if .Values.config.database.passwordExistingSecret.enabled }} + {{- /* [PARITY] Both are template-evaluated so the defaults in + values.yaml can follow the secret the chart actually + renders. `name` was already tpl'd; `key` was not, and had + to be, once its default became + `{{ .Values.postgresql.auth.secretKeys.adminPasswordKey }}`. + + Why the defaults changed: they were hardcoded to + `-postgresql` / `postgres-password`, so setting + postgresql.auth.existingSecret brought PostgreSQL up + against the new secret while Nebraska kept looking for a + secret the chart no longer rendered, and sat in + CreateContainerConfigError. Pre-existing in 2.0.0, but + 3.0.0 advertises existingSecret, so it had to work. */}} name: {{ tpl .Values.config.database.passwordExistingSecret.name . }} - key: {{ .Values.config.database.passwordExistingSecret.key }} + key: {{ tpl .Values.config.database.passwordExistingSecret.key . }} {{- else }} name: {{ include "nebraska.fullname" . }} key: dbPassword @@ -212,7 +247,6 @@ spec: value: {{ $value | quote }} {{- end }} {{- if .Values.extraEnv }} - # Extra environment variables {{- toYaml .Values.extraEnv | nindent 12 }} {{- end }} {{- if .Values.config.hostFlatcarPackages.enabled }} diff --git a/charts/nebraska/templates/postgresql.yaml b/charts/nebraska/templates/postgresql.yaml new file mode 100644 index 000000000..7d579a686 --- /dev/null +++ b/charts/nebraska/templates/postgresql.yaml @@ -0,0 +1,407 @@ +{{- /* + The PostgreSQL that chart 2.0.0 got from the Bitnami postgresql 11.9.1 + subchart. See Chart.yaml for why it is vendored rather than depended on. + + This file renders the same five objects the subchart did, under the same + names: ServiceAccount, Secret, Service, headless Service, StatefulSet. + Names, labels and the `postgres-password` secret key are deliberately + kept exactly the same as the subchart's output, so that existing releases, external + secret managers and `config.database.passwordExistingSecret` keep working. + + Scope of each block is marked so a reviewer can tell what the migration + required from what it merely made a good moment to fix: + + [MIGRATION] required to run the official image at all. + [PARITY] restores behaviour the subchart had, which would otherwise + vanish silently when the subchart was removed. + [GUARD] new, and specifically about making this migration safe. + [EXTRA] a pre-existing bug or gap, fixed while in the file. None of + these are needed for the migration; they are called out + individually and can be dropped without affecting it. +*/ -}} +{{- /* [GUARD] validateUnknownValues runs unconditionally: stale Bitnami values + must be reported even when postgresql.enabled is false, since that is + exactly when nobody would notice them being ignored. validateImage, + validateMountPath and validateDataDirMigration are gated on + postgresql.enabled, they guard things that only exist when the + bundled database runs. */ -}} +{{- include "nebraska.postgresql.validateUnknownValues" . -}} +{{- include "nebraska.postgresql.validateImage" . -}} +{{- include "nebraska.postgresql.validateMountPath" . -}} +{{- include "nebraska.postgresql.validateDataDirMigration" . -}} +{{- if .Values.postgresql.enabled }} +{{- $pgName := include "nebraska.postgresql.fullname" . -}} +{{- $pgHl := include "nebraska.postgresql.headlessName" . -}} +{{- $pgPort := int .Values.postgresql.service.port -}} +{{- $user := .Values.postgresql.auth.username -}} +{{- $db := .Values.postgresql.auth.database -}} +{{- $persistence := .Values.postgresql.primary.persistence -}} +{{- if .Values.postgresql.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nebraska.postgresql.serviceAccountName" . }} + {{- include "nebraska.postgresql.metadata" . | nindent 2 }} +automountServiceAccountToken: {{ .Values.postgresql.serviceAccount.automountServiceAccountToken }} +--- +{{- end }} +{{- if not .Values.postgresql.auth.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $pgName }} + labels: + {{- include "nebraska.postgresql.labels" . | nindent 4 }} + {{- with .Values.extraLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- /* [EXTRA, not required by the migration] This was already a problem before, but + the migration makes an uninstall/reinstall cycle much more likely, so + it is fixed here. The PVC outlives the release (the StatefulSet + controller owns it, and + StatefulSet deletion does not cascade to PVCs). If the Secret does not + outlive it too, an uninstall/reinstall cycle regenerates the password + while the old database on the surviving volume still expects the old + one. Then nobody can log in any more, unless you kept a copy of the old + password. + + Consequence to be aware of: the Secret therefore survives + `helm uninstall`. Installing chart 2.0.0 over the leftovers aborts in + Bitnami's own check with + + PASSWORDS ERROR ... does not contain the key "password" + + because 3.0.0 emits only `postgres-password`, while 2.0.0's subchart + also expected a `password` key (a SEPARATE app-user credential in + that chart). + + We decided to accept that downgrade failure instead of writing a fake key. + The alternative was to also emit a `password` key so the Secret keeps + 2.0.0's shape, which would make the downgrade work, but that key + would hold the SUPERUSER password under a name that meant "app user" + in 2.0.0. A value that is wrong about itself, and that anything + reading the old key would silently misuse, is worse than a documented + failure with a clear message. If we ever change our mind about this, the + scope is this data block plus a README note. */}} + helm.sh/resource-policy: keep + {{- with .Values.extraAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- /* [PARITY] Password precedence, restoring what the Bitnami subchart did: + + 1. the value already in the cluster (rotations survive upgrades) + 2. an explicit postgresql.auth.postgresPassword + 3. a freshly generated random password + + Bitnami defaulted postgresPassword to "" and generated with + randAlphaNum; chart 2.0.0 overrode that with the literal "changeIt", + which shipped a published password for a superuser reachable from the + whole pod network. Generating restores the upstream behaviour and keeps + `helm install` working with no flags. + + lookup returns empty during `helm template` and `--dry-run`, so a + rendered manifest shows a fresh random value each time. That is fine for + real installs but shows as drift under GitOps, those users should set + postgresql.auth.existingSecret. */}} + {{- $key := .Values.postgresql.auth.secretKeys.adminPasswordKey }} + {{- $existing := (lookup "v1" "Secret" .Release.Namespace $pgName) | default dict }} + {{- $current := index ($existing.data | default dict) $key | default "" }} + {{- if $current }} + {{ $key }}: {{ $current | quote }} + {{- else if .Values.postgresql.auth.postgresPassword }} + {{ $key }}: {{ .Values.postgresql.auth.postgresPassword | toString | b64enc | quote }} + {{- else }} + {{ $key }}: {{ randAlphaNum 24 | b64enc | quote }} + {{- end }} +--- +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $pgName }} + {{- include "nebraska.postgresql.metadata" . | nindent 2 }} +spec: + type: ClusterIP + sessionAffinity: None + ports: + - name: tcp-postgresql + port: {{ $pgPort }} + targetPort: tcp-postgresql + protocol: TCP + selector: + {{- include "nebraska.postgresql.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $pgHl }} + {{- include "nebraska.postgresql.metadata" . | nindent 2 }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: tcp-postgresql + port: {{ $pgPort }} + targetPort: tcp-postgresql + protocol: TCP + selector: + {{- include "nebraska.postgresql.selectorLabels" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $pgName }} + {{- include "nebraska.postgresql.metadata" . | nindent 2 }} +spec: + {{- /* [MIGRATION] Hardcoded, not a value. A second replica would mount the same + single PVC and run a second postmaster over one data directory, which + corrupts it. PostgreSQL replicas are a different StatefulSet with their own + volumes and a replication protocol, not `replicas: 2`, and Nebraska has no + read/write split to use them anyway. See _validate.tpl for the guidance + given to anyone who sets postgresql.architecture. */}} + replicas: 1 + serviceName: {{ $pgHl }} + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "nebraska.postgresql.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "nebraska.postgresql.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "nebraska.postgresql.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.postgresql.serviceAccount.automountServiceAccountToken }} + terminationGracePeriodSeconds: {{ .Values.postgresql.terminationGracePeriodSeconds }} + {{- /* [PARITY] The subchart took pullSecrets as BARE STRINGS + (`pullSecrets: [regcred]`) and converted them to + LocalObjectReference itself. Emitting them with toYaml produced + `- regcred`, which the apiserver rejects. Accept both the old + string form and the object form used elsewhere in this chart. */}} + {{- with .Values.postgresql.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + {{- if kindIs "string" . }} + - name: {{ . }} + {{- else }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- end }} + securityContext: + {{- toYaml .Values.postgresql.podSecurityContext | nindent 8 }} + containers: + - name: postgresql + image: {{ include "nebraska.image" ( dict "imageRoot" .Values.postgresql.image "global" .Values.global "context" nil ) }} + imagePullPolicy: {{ .Values.postgresql.image.pullPolicy }} + securityContext: + {{- toYaml .Values.postgresql.containerSecurityContext | nindent 12 }} + {{- /* [PARITY] Replaces the subchart's primary.configuration / + extendedConfiguration / existingConfigmap, which were the only + ways to set server options and all disappeared with it. + This is the only declarative, first-boot way to set a + postmaster start-time GUC (ALTER SYSTEM works but needs + superuser access plus a restart, and its postgresql.auto.conf + dies with the volume when persistence is off), and the image + has no env var for it, so without this there is no managed + route to logical replication, TLS, or connection logging. + The entrypoint dispatches on $1 == "postgres", so the first + element must be "postgres". */}} + {{- with .Values.postgresql.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "nebraska.postgresql.secretName" . }} + key: {{ .Values.postgresql.auth.secretKeys.adminPasswordKey }} + - name: POSTGRES_USER + value: {{ $user | quote }} + - name: POSTGRES_DB + value: {{ $db | quote }} + {{- /* [MIGRATION] PGDATA must be a SUBDIRECTORY of the volume mount point. PostgreSQL + refuses to start unless the data directory is exactly 0700 or 0750 + (see src/backend/utils/init/miscinit.c), and a mounted volume root + is neither: the docker-library image ships it 1777 so that arbitrary + --user values work, fsGroup adds group-write, and ext4 volumes also + arrive with a lost+found entry that makes initdb refuse to run. + Letting initdb create its own subdirectory sidesteps all three. */}} + - name: PGDATA + value: {{ printf "%s/%s" .Values.postgresql.dataMountPath .Values.postgresql.dataSubdir | quote }} + - name: PGPORT + value: {{ $pgPort | quote }} + {{- with .Values.postgresql.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: tcp-postgresql + containerPort: {{ $pgPort }} + protocol: TCP + {{- /* [EXTRA, not required by the migration] This fixes a + pre-existing bug: chart 2.0.0 had the same problem with the + Bitnami image and no preStop hook. Dropping this block does not + affect the migration; it only restores the old + wait-then-SIGKILL shutdown behaviour. */}} + # Kubernetes sends SIGTERM on pod termination. Whether the image's + # `STOPSIGNAL SIGINT` is honoured is runtime-dependent, containerd + # and CRI-O do respect it, but it is not guaranteed by the Kubernetes + # API, so a chart cannot rely on it. If SIGTERM does reach PostgreSQL + # it means "smart shutdown": refuse new connections but wait + # indefinitely for existing ones to end. Nebraska holds a pool of up to + # 25 connections that do not close on their own, so the pod would sit + # until terminationGracePeriodSeconds expires and then be SIGKILLed, + # leaving the cluster to do crash recovery on the next start. `-m fast` + # rolls back open transactions and shuts down cleanly, quickly for a + # database of this size; a big shared_buffers flush can need longer, + # which is what terminationGracePeriodSeconds bounds. + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + {{- /* -W, not the absence of -w. Waiting is `pg_ctl stop`'s + DEFAULT ("-w, --wait ... (default)"), so dropping -w + changes nothing, opting out requires -W explicitly. + That matters because the postmaster is PID 1: the moment + it exits, the runtime tears down the sandbox and SIGKILLs + a still-waiting hook. It exits 137 and the kubelet + records a FailedPreStopHook Warning on EVERY pod + deletion, even though the shutdown itself is clean. + With -W: "server shutting down", exit 0, the shutdown + completes in the background, and the next start shows no + crash recovery. terminationGracePeriodSeconds is what + gives it room to finish. */}} + - exec pg_ctl -D "$PGDATA" -m fast -W stop + {{- /* [EXTRA, not required by the migration] Chart 2.0.0 shipped + these probe numbers plus a liveness initialDelaySeconds: 30 + (dropped here, the startupProbe covers the delay), and no + startupProbe, so this is a pre-existing hazard rather than one + the migration introduced. Fixed here because a first-time + initdb is exactly what every migrated install is about to do. + + First start has to cover a recursive fsGroup chown of the whole + volume, an fsync-heavy initdb, and the entrypoint's socket-only + temporary server, during which pg_isready on 127.0.0.1 cannot + succeed. Without a startupProbe the liveness budget was ~80s; if + it fired mid-initdb the data directory was left non-empty without + PG_VERSION, which initdb then refuses forever: a permanent + CrashLoopBackOff needing manual cleanup. */}} + startupProbe: + {{- include "nebraska.postgresql.probeCommand" . | nindent 12 }} + {{- toYaml .Values.postgresql.startupProbe | nindent 12 }} + livenessProbe: + {{- include "nebraska.postgresql.probeCommand" . | nindent 12 }} + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + {{- include "nebraska.postgresql.probeCommand" . | nindent 12 }} + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + resources: + {{- toYaml .Values.postgresql.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.postgresql.dataMountPath }} + {{- /* [MIGRATION] The image patches DEFAULT_PGSOCKET_DIR to /var/run/postgresql, so + the unix socket and its lock file land there. (postmaster.pid is + always inside PGDATA, not here.) Without a writable mount the + server cannot create its socket and does not start at all, so this + is mandatory whenever readOnlyRootFilesystem is true, and it is + also what makes `kubectl exec ... pg_dump` over the socket work. */}} + - name: run + mountPath: /var/run/postgresql + {{- /* [MIGRATION] Only strictly required by the entrypoint's nss_wrapper path, which + runs mktemp when the effective uid is not in /etc/passwd. That happens + when podSecurityContext.runAsUser is set to something other than + the image's own postgres uid. Cheap insurance, so it is always mounted. */}} + - name: tmp + mountPath: /tmp + {{- /* [PARITY] The subchart mounted this too (shmVolume, default on). + The container runtime default for /dev/shm is 64Mi, which PostgreSQL + can exhaust ("could not resize shared memory segment"). */}} + - name: dshm + mountPath: /dev/shm + {{- with .Values.postgresql.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- /* The single escape hatch, and deliberately the only one. + + Chart 2.0.0 documented ten postgresql.* parameters. Everything else + the Bitnami subchart accepted was reachable but never offered, + tested or supported by Nebraska, so re-implementing it here would be + promising something new instead of keeping what was already there. + + Merged into the pod spec, so it covers the cases that do come up -- + an initContainer to chown the volume where fsGroup is ignored, + priorityClassName, schedulerName, hostAliases, + topologySpreadConstraints, without a value per field. + + Do not set `containers` here: it would emit a second containers key + alongside the chart's own. Anything needing a sidecar wants a real + database, not this one. */}} + {{- with .Values.postgresql.extraPodSpec }} + {{- toYaml . | nindent 6 }} + {{- end }} + volumes: + - name: run + emptyDir: {} + - name: tmp + emptyDir: {} + - name: dshm + emptyDir: + medium: Memory + {{- with .Values.postgresql.shmSizeLimit }} + {{- /* Memory-backed emptyDir is charged to the pod's working set and + otherwise defaults to half the node's RAM. Unbounded, plus no + memory limit, is a node-pressure eviction vector. */}} + sizeLimit: {{ . }} + {{- end }} + {{- if not $persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.postgresql.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $persistence.enabled }} + volumeClaimTemplates: + - metadata: + {{- /* [MIGRATION] Named `data` because the Bitnami subchart named it `data`. + The StatefulSet controller derives the PVC name from this + (`data--0`), so changing it would strand the existing volume and + silently provision an empty one. volumeClaimTemplates is also an + immutable field, so this must match for `helm upgrade` to be accepted + at all. */}} + name: data + {{- with $persistence.labels }} + labels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- toYaml $persistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ $persistence.size | quote }} + {{- with (include "nebraska.postgresql.storageClass" .) }} + {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/nebraska/values.yaml b/charts/nebraska/values.yaml index 7bedac110..f15cf6e98 100644 --- a/charts/nebraska/values.yaml +++ b/charts/nebraska/values.yaml @@ -74,15 +74,21 @@ config: database: host: "" - port: 5432 + # Empty follows the bundled database's port (postgresql.service.port) when + # postgresql.enabled, and 5432 for an external host. Set explicitly to pin. + port: "" dbname: '{{ .Values.postgresql.auth.database }}' username: '{{ .Values.postgresql.auth.username }}' password: "" sslMode: "" passwordExistingSecret: enabled: true - name: '{{ .Release.Name }}-postgresql' - key: postgres-password + # Follows postgresql.auth.existingSecret / nameOverride / fullnameOverride. + # Hardcoding '{{ .Release.Name }}-postgresql' here meant that setting + # postgresql.auth.existingSecret brought PostgreSQL up against the new + # secret while Nebraska kept looking for the old one and never started. + name: '{{ include "nebraska.postgresql.secretName" . }}' + key: '{{ .Values.postgresql.auth.secretKeys.adminPasswordKey }}' extraArgs: [] # - "-http-log" @@ -154,7 +160,7 @@ ingress: ingressClassName: "" update: # Create a separate ingress for the /v1/update and /flatcar paths, - # with it's own annotations. + # with its own annotations. enabled: false annotations: {} ingressClassName: "" @@ -198,30 +204,271 @@ readinessProbe: extraLabels: {} extraAnnotations: {} -# Configuration values for the postgresql dependency. -# ... for more options see https://github.com/bitnami/charts/tree/master/bitnami/postgresql +# Configuration values for the bundled PostgreSQL StatefulSet. +# +# WHAT THIS IS. A deliberately minimal, single-replica PostgreSQL whose only job +# is to make `helm install nebraska` produce a working Nebraska. It does no +# backups, no failover and no automated major-version upgrades. That is the same +# scope the Bitnami subchart had here, 11.9.1 shipped no backup automation +# either, and chart 2.0.0 configured none. For anything you care about, set +# `postgresql.enabled: false` and point `config.database.*` at a database you +# operate, or at an operator such as CloudNativePG. +# +# HOW THE KEYS WERE CHOSEN. Names that existed in 2.0.0 keep their meaning +# (`enabled`, `auth.*`, `image.*`, `primary.persistence.*`, `serviceAccount.*`, +# `nameOverride`), so most values files keep working. Keys that lived under +# `primary.*` purely because the subchart was a separate chart moved up to +# `postgresql.*`. Anything this chart does not read that would have changed +# behaviour is reported at render time rather than ignored, values left at +# inert Bitnami defaults are accepted silently; see templates/_validate.tpl. postgresql: enabled: true + # Upgrading from chart 2.0.0 with persistence enabled moves the data directory + # inside the volume, which PostgreSQL cannot follow: it would initialise a new + # empty cluster next to the old one, with no error. The chart refuses to render + # until you confirm you have either migrated the data or intend to start empty. + # See "Upgrading to 3.0.0" in the README. Ignored when persistence is disabled. + acknowledgeDataDirMigration: false + + # Overrides the "postgresql" component name used for the Services (main and + # headless), Secret, ServiceAccount and StatefulSet (`-`). + # As in the Bitnami subchart, the suffix is skipped when the release name + # already contains this value. + nameOverride: "" + # Replaces the generated name outright. + fullnameOverride: "" + auth: + # The official postgres image has no separate superuser/app-user split the + # way the Bitnami image did: POSTGRES_USER *is* the superuser created by + # initdb, and POSTGRES_DB is created owned by it. username: postgres database: nebraska - postgresPassword: changeIt + # Empty means "generate a random password on first install and keep it". + # This is what the Bitnami subchart did; chart 2.0.0 overrode it with the + # literal "changeIt", which published the superuser password of a database + # reachable from the entire pod network. + # + # Retrieve the generated password with: + # kubectl get secret -postgresql \ + # -o jsonpath='{.data.postgres-password}' | base64 -d + # + # Set it explicitly if you manage credentials yourself, or use + # existingSecret. GitOps users should prefer existingSecret: generation is + # not deterministic, so a re-render shows drift. + postgresPassword: "" + # Use a pre-existing secret instead of rendering one from postgresPassword. + # Evaluated as a template. The key is taken from secretKeys.adminPasswordKey. + existingSecret: "" + secretKeys: + adminPasswordKey: postgres-password + image: registry: docker.io - repository: bitnamilegacy/postgresql - tag: 17.5.0 + repository: postgres + # Debian bookworm, not Alpine, and deliberately so. + # + # bookworm carries glibc 2.36, the same C library as the Bitnami image + # this chart replaces, so text collation, and therefore `ORDER BY` results + # and btree index ordering, are unchanged by the migration. + # + # The Alpine variants are smaller (~112MiB vs ~154MiB compressed) but use + # musl, whose collation is basically byte order. Worse, musl makes up a + # locale for any name it is given and PostgreSQL's own collation-version + # guard is compiled only for glibc, so attaching a glibc-era volume to an + # Alpine image starts cleanly and silently returns wrong index results. + # For a database, 40MiB is not worth that failure mode. + # + # Pinned to the PostgreSQL 17 line to match the previous default. Do not + # move to 18 without reading the migration notes: 18 relocates both PGDATA + # and the declared volume, and a major-version jump needs pg_upgrade or a + # dump/restore either way. + tag: 17-bookworm + # [PARITY] The subchart had image.digest too. + # Pin by content instead of by tag. When set this replaces the tag entirely, + # so a rebuild of `17-bookworm` cannot change what runs. Recommended for + # production; keep it updated with Renovate/Dependabot or you will pin + # yourself to an unpatched image, which is the problem this chart set out to + # solve. Example: sha256:abc123... + digest: "" + # Note: with a floating tag and IfNotPresent, a node that has already cached + # the tag will not pick up a rebuilt image. Use a digest, or set Always, if + # you need CVE fixes to actually reach your nodes. + pullPolicy: IfNotPresent + pullSecrets: [] + + service: + port: 5432 + + # Where the data volume is mounted, and the subdirectory inside it that + # becomes PGDATA. + # + # dataMountPath must NOT be an ancestor of the image's declared VOLUME + # (/var/lib/postgresql/data on 17, /var/lib/postgresql on 18). Mount above it + # and the runtime lays an empty volume over the top, hiding everything your PVC + # holds underneath, observed live: a Bitnami data directory present on the PV + # and unreadable from inside the pod. The chart refuses that configuration. + # + # These must stay two separate settings: PostgreSQL refuses to start unless + # PGDATA is exactly 0700/0750, which a volume mount root never is. See the + # comment in templates/postgresql.yaml. + dataMountPath: /var/lib/postgresql/data + dataSubdir: pgdata + primary: persistence: + # Default OFF, inherited unchanged from chart 2.0.0. + # + # Kept off deliberately: this database is a convenience for getting + # started, and defaulting to a PVC would leave orphaned volumes behind + # every `helm uninstall` for evaluation installs. Anyone running Nebraska + # for real should either turn this on or, better, use an external + # database. + # + # Note if you DO turn it on: the data directory layout changed in 3.0.0, + # so upgrading a persistent 2.0.0 install needs a dump and restore. The + # chart refuses to render such an upgrade until you acknowledge it, see + # acknowledgeDataDirMigration above. enabled: false + # Empty means "use the cluster default StorageClass". `global.storageClass` + # is honoured as a fallback, and the Bitnami "-" sentinel (meaning "render + # an empty storageClassName") still works, both for parity with 2.0.0. storageClass: accessModes: - ReadWriteOnce size: 1Gi + labels: {} + annotations: {} + serviceAccount: create: true + # Defaults to the generated name. Honoured whether or not create is true, + # matching the Bitnami subchart. + name: "" automountServiceAccountToken: false + # uid/gid 999 is the "postgres" user in the Debian-based images. The Alpine + # variants use 70 instead, change all three together if you switch + # image.tag to an Alpine flavour, or the data directory ends up owned by the + # wrong user and PostgreSQL refuses to start. + podSecurityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + # [EXTRA, not required by the migration] Performance only. + # Without this the kubelet recursively chowns the entire volume on every + # mount, which on a large database adds minutes to every pod start. + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # [EXTRA, not required by the migration] Supports the preStop hook, which + # fixes a pre-existing unclean-shutdown bug. 2.0.0 set nothing, so pods got + # the Kubernetes default of 30, with no hook. + # + # The hook signals a fast shutdown and returns immediately; this budget is what + # PostgreSQL then has to finish flushing before the kubelet sends SIGKILL. A + # small database needs a fraction of a second, a large one with a big + # shared_buffers can need much longer, raise it if you see the next start log + # "database system was not properly shut down". + terminationGracePeriodSeconds: 90 + + # [PARITY] Replaces the subchart's primary.configuration / existingConfigmap. + # Extra arguments for the postgres server. The first element must be + # "postgres", the image entrypoint dispatches on it. This is the only + # declarative, first-boot way to set start-time GUCs (ALTER SYSTEM works, but + # needs superuser access plus a manual restart, and its postgresql.auto.conf + # is lost with the volume when persistence is off). For example: + # args: + # - postgres + # - "-c" + # - "log_connections=on" # connection auditing + # - "-c" + # - "wal_level=logical" # required to publish logical replication + args: [] + + # [PARITY] The subchart had shmVolume.sizeLimit; this replaces it. + # Bound the memory-backed /dev/shm emptyDir. Empty means unbounded, which + # defaults to half the node's RAM and is charged to this pod. + shmSizeLimit: 256Mi + + # [EXTRA, not required by the migration] 2.0.0 had no startupProbe. + # First-start budget. Covers the fsGroup chown of the volume, initdb, and the + # entrypoint's socket-only temporary server. 5s x 60 = 300s. + startupProbe: + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + + # Requests only, no limits, matching chart 2.0.0 and the Bitnami subchart, + # which also shipped `limits: {}`. + # + # A memory limit is NOT set on purpose. PostgreSQL's working set depends + # entirely on your data and query pattern, and a limit that is too low does not + # degrade gracefully: the kernel OOM-kills the postmaster mid-transaction and + # the next start does crash recovery. The chart will not guess a number. Set + # `postgresql.resources.limits` once you have measured your instance. + # + # The specific resource-exhaustion risk that a memory-backed volume creates is + # handled separately, by bounding /dev/shm (see shmSizeLimit). + resources: + requests: + cpu: 250m + memory: 256Mi + + # Env and volumes stay first-class: they are the documented answers to two + # things the official image does not do for us, initdb scripts (mount a + # ConfigMap at /docker-entrypoint-initdb.d) and TLS certificates. + extraEnv: [] + extraVolumes: [] + extraVolumeMounts: [] + + # The single escape hatch, merged into the PostgreSQL pod spec. + # + # Chart 2.0.0 documented ten postgresql.* parameters. Everything else the + # Bitnami subchart happened to accept, sidecars, LDAP, TLS termination, + # replicas, an exporter, its own NetworkPolicy, was never offered, tested or + # documented by Nebraska, and most of it does not make sense for a single + # bundled convenience database anyway. Re-implementing it here would be + # promising something new, not keeping what was there. So there is one escape + # hatch instead of a + # value per Bitnami feature. + # + # It covers the cases that genuinely do come up, for example chowning the + # volume where the storage driver ignores fsGroup (what Bitnami's + # volumePermissions did): + # + # extraPodSpec: + # initContainers: + # - name: volume-permissions + # image: busybox:1.36 + # command: ['sh','-c','chown -R 999:999 /var/lib/postgresql/data'] + # # runAsNonRoot: true is set at pod level in this chart; the + # # container-level override is required or the kubelet rejects the pod. + # securityContext: { runAsUser: 0, runAsNonRoot: false } + # volumeMounts: + # - name: data + # mountPath: /var/lib/postgresql/data + # + # and scheduling knobs: priorityClassName, schedulerName, hostAliases, + # topologySpreadConstraints. + # + # Do NOT set `containers` here, it emits a second containers key next to the + # chart's own. If you need a sidecar (a metrics exporter, a backup agent) you + # have outgrown the bundled database: set postgresql.enabled=false and run a + # real one. An exporter does not need to share the pod in any case; point it at + # the Service. + extraPodSpec: {} + extraObjects: [] # - apiVersion: external-secrets.io/v1beta1 # kind: ExternalSecret