diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 837d5f3a..ca919fa0 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -74,7 +74,9 @@ jobs: oci://registry.developers.crunchydata.com/crunchydata/pgo \ --version ${{ env.PGO_VERSION }} - - name: helm render/install eoapi templates + - id: helm-render-install-eoapi-templates + name: helm render/install eoapi templates + continue-on-error: true run: | export GITSHA='${{github.sha}}' @@ -87,18 +89,53 @@ jobs: -f ./eoapi/test-k3s-unittest-values.yaml \ ./eoapi - - name: sleep for 10s seconds while services boot - shell: bash - run: sleep 10s + exit $? -# - name: Setup upterm session -# uses: lhotari/action-upterm@v1 + - name: debug pgstac-migrate job failure + if: steps.helm-render-install-eoapi-templates.outcome == 'failure' + continue-on-error: true + run: | + echo "Extracting pgstac-migrate job info and logs for debugging..." + + # Get job details + echo "===== pgstac-migrate Job Details =====" + kubectl get job pgstac-migrate -o yaml || echo "Could not get pgstac-migrate job details" + + # Get pod details + echo "===== pgstac-migrate Pod Details =====" + kubectl get pods -l app=pgstac-migrate --all-namespaces || echo "Could not find pgstac-migrate pods" + + # Extract logs from pgstac-migrate pod(s) + echo "===== pgstac-migrate Pod Logs =====" + echo "Looking for completed pods from pgstac-migrate job..." + COMPLETED_PODS=$(kubectl get pods --selector=job-name=pgstac-migrate --field-selector=status.phase=Succeeded,status.phase=Failed -o jsonpath='{.items[*].metadata.name}' 2>/dev/null) + if [ -n "$COMPLETED_PODS" ]; then + echo "Found completed pods from pgstac-migrate job. Extracting logs from each:" + for POD in $COMPLETED_PODS; do + echo "--- Logs from completed pod $POD ---" + kubectl logs pod/$POD || echo "Could not get logs from pod $POD" + done + else + echo "No completed pods found for pgstac-migrate job" + fi + + # Get details about the database pods/services + echo "===== Database Pod/Service Details =====" + # Find database service + kubectl get svc | grep "db" || echo "Could not find database services" + # Find database pods + kubectl get pods | grep "db-" || echo "Could not find database pods" + + # Check for any events related to the job or pods + echo "===== Related Kubernetes Events =====" + kubectl get events | grep -E "pgstac|db" || echo "No relevant events found" - id: watchservices name: watch services boot - timeout-minutes: 10 + timeout-minutes: 3 continue-on-error: true run: | + # Now wait for services to boot up while [[ -z "$(kubectl get pod | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do echo "still waiting for raster service to start..." sleep 1 @@ -124,14 +161,6 @@ jobs: echo "===== Pod Status =====" kubectl get pods - # Extract logs from database pod - echo "===== Database Pod Logs =====" - kubectl get pod | grep "^db-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get database logs" - - # Extract logs from pgstacbootstrap pod - echo "===== PGSTACBootstrap Pod Logs =====" - kubectl get pod | grep "^pgstacbootstrap-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get pgstacbootstrap logs" - # Extract logs from raster pod init container (wait-for-pgstacbootstrap) echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" @@ -161,7 +190,7 @@ jobs: - name: run the tests id: testrunner - continue-on-error: true + # continue-on-error: true run: | kubectl get svc --all-namespaces kubectl get ingress --all-namespaces -o jsonpath='{range .items[0]}kubectl describe ingress {.metadata.name} -n {.metadata.namespace}{end}' | sh @@ -192,14 +221,6 @@ jobs: echo "===== Pod Status =====" kubectl get pods - # Extract logs from database pod - echo "===== Database Pod Logs =====" - kubectl get pod | grep "^db-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get database logs" - - # Extract logs from pgstacbootstrap pod - echo "===== PGSTACBootstrap Pod Logs =====" - kubectl get pod | grep "^pgstacbootstrap-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get pgstacbootstrap logs" - # Extract logs from raster pod init container (wait-for-pgstacbootstrap) echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" @@ -225,4 +246,4 @@ jobs: - name: helm uninstall eoapi templates run: | - helm uninstall $RELEASE_NAME \ No newline at end of file + helm uninstall $RELEASE_NAME diff --git a/helm-chart/eoapi/README.md b/helm-chart/eoapi/README.md new file mode 100644 index 00000000..f24990e7 --- /dev/null +++ b/helm-chart/eoapi/README.md @@ -0,0 +1,141 @@ +# EOAPI Helm Chart + +This Helm chart deploys the EOAPI (Earth Observation API) stack, which includes STAC API, raster tile services, vector tile services, and a multidimensional data service. + +## Overview + +The chart sets up: + +- A PostgreSQL database with PostGIS and PgSTAC extensions +- STAC API service for metadata discovery and search +- Titiler for raster tile services +- TIPG for vector tile services +- Optional multidimensional data service + +## Prerequisites + +- Kubernetes 1.16+ +- Helm 3.0+ +- PV provisioner support in the underlying infrastructure +- CrunchyData Postgres Operator (for the PostgreSQL database) + +## Installation + +```bash +# Install Postgres Operator first +helm install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo + +# Then install eoapi +helm install eoapi ./eoapi +``` + +## Configuration + +The chart can be configured via `values.yaml`. See the chart's `values.yaml` file for all available options and detailed descriptions. + +Key configuration sections: + +```yaml +# Services to enable +apiServices: + - raster + - stac + - vector + # - multidim (disabled by default) + +# Ingress configuration +ingress: + enabled: true + className: "nginx" + # ... + +# Database configuration +postgrescluster: + enabled: true + # ... +``` + +## PgSTAC Bootstrap Process + +The chart includes a streamlined process for initializing and setting up the PgSTAC database. + +### PgSTAC Bootstrap Overview + +The setup process consists of two main jobs: + +1. **pgstac-migrate job**: Runs the pypgstac migrate command to initialize the database schema, applies settings, and sets necessary permissions. +2. **pgstac-load-samples job**: (Optional) Loads sample STAC data only when sample loading is enabled. + +### Improvements in PgSTAC Bootstrap + +- Replaced custom Python script with pypgstac migrate command +- Moved SQL settings to a dedicated SQL file for better maintainability +- Separated sample data loading into an optional job +- Uses standard PostgreSQL environment variables +- Ensures the process remains idempotent for safe re-runs + +### PgSTAC Directory Structure + +The codebase has been reorganized to separate different types of files: + +- `initdb-data/settings/`: Contains configuration settings like the PgSTAC settings SQL file +- `initdb-data/samples/`: Contains sample data files that are loaded only when sample loading is enabled + +### PgSTAC Configuration + +- Enable/disable the setup process through `pgstacBootstrap.enabled` +- Control sample data loading: + - New approach: `pgstacBootstrap.settings.loadSamples` (recommended) + - Legacy approach: `pgstacBootstrap.settings.envVars.LOAD_FIXTURES` (deprecated) + +Example configuration: + +```yaml +pgstacBootstrap: + enabled: true + settings: + # General configuration options + loadSamples: true # Set to false to disable sample data loading + + resources: + requests: + cpu: "512m" + memory: "1024Mi" + limits: + cpu: "512m" + memory: "1024Mi" +``` + +## Services + +### STAC API + +The STAC API service provides a standardized way to search and discover geospatial data. + +### Raster Services (Titiler) + +Provides dynamic tiling for raster data through the TiTiler implementation. + +### Vector Services (TIPG) + +Provides vector tile services for PostGIS data through the TIPG implementation. + +### Multidimensional Services (Optional) + +Provides services for multidimensional data (time series, etc.). + +## Persistence + +The chart uses PostgreSQL for data persistence. Make sure to configure appropriate storage for production use. + +## Upgrading + +When upgrading the chart, consider any changes to values.yaml and migrations that might need to be applied. + +## Uninstallation + +```bash +helm delete eoapi +``` + +Note that PVs may need to be manually deleted if you want to remove all data. diff --git a/helm-chart/eoapi/initdb-data/pgstac-setup.py b/helm-chart/eoapi/initdb-data/pgstac-setup.py deleted file mode 100644 index b05b5349..00000000 --- a/helm-chart/eoapi/initdb-data/pgstac-setup.py +++ /dev/null @@ -1,60 +0,0 @@ -#! /usr/bin/env python3 - -# This script is used to setup the pgstac database. -# It is run as a job in the pgstacbootstrap pod. -# It is important that this script and all of its steps are idempotent. - -import os -import psycopg -from psycopg import sql -from psycopg.conninfo import make_conninfo -from pypgstac.db import PgstacDB -from pypgstac.migrate import Migrate - -admin_db_conninfo = make_conninfo(os.environ['PGADMIN_URI']) -print("[ REGISTER ]: postgis") -with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;")) - -pgdb = PgstacDB(dsn=os.environ['PGADMIN_URI'], debug=True) -print(f"[ VERSION ]: {pgdb.version=}") -Migrate(pgdb).run_migration(pgdb.version) - -with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: - with conn.cursor() as cur: - # NOTE: most of these should've been set up by postgresql operator - # see `helm-chart/eoapi/values.yaml:postgrescluster` but in case - # they haven't been - cur.execute( - sql.SQL( - "GRANT CONNECT ON DATABASE {db_name} TO {username};" - "GRANT CREATE ON DATABASE {db_name} TO {username};" # Allow schema creation - "GRANT USAGE ON SCHEMA public TO {username};" - "ALTER DEFAULT PRIVILEGES IN SCHEMA public " - "GRANT ALL PRIVILEGES ON TABLES TO {username};" - "ALTER DEFAULT PRIVILEGES IN SCHEMA public " - "GRANT ALL PRIVILEGES ON SEQUENCES TO {username};" - "GRANT pgstac_read TO {username};" - "GRANT pgstac_ingest TO {username};" - "GRANT pgstac_admin TO {username};" - "ALTER USER {username} SET search_path TO pgstac, public;" # add pgstac to search_path by default - ).format( - db_name=sql.Identifier(os.environ["POSTGRES_DBNAME"]), - username=sql.Identifier(os.environ["POSTGRES_USER"]), - ) - ) - -with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute( - sql.SQL( - "INSERT INTO pgstac.pgstac_settings (name, value) " - " VALUES " - " ('context', 'auto')," - " ('context_estimated_count', '100000')," - " ('context_estimated_cost', '100000')," - " ('context_stats_ttl', '1 day')" - " ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;" - ) - ) diff --git a/helm-chart/eoapi/initdb-data/my_data.sql b/helm-chart/eoapi/initdb-data/samples/my_data.sql similarity index 100% rename from helm-chart/eoapi/initdb-data/my_data.sql rename to helm-chart/eoapi/initdb-data/samples/my_data.sql diff --git a/helm-chart/eoapi/initdb-data/noaa-emergency-response.json b/helm-chart/eoapi/initdb-data/samples/noaa-emergency-response.json similarity index 100% rename from helm-chart/eoapi/initdb-data/noaa-emergency-response.json rename to helm-chart/eoapi/initdb-data/samples/noaa-emergency-response.json diff --git a/helm-chart/eoapi/initdb-data/noaa-eri-nashville2020.json b/helm-chart/eoapi/initdb-data/samples/noaa-eri-nashville2020.json similarity index 100% rename from helm-chart/eoapi/initdb-data/noaa-eri-nashville2020.json rename to helm-chart/eoapi/initdb-data/samples/noaa-eri-nashville2020.json diff --git a/helm-chart/eoapi/initdb-data/settings/pgstac-settings.sql b/helm-chart/eoapi/initdb-data/settings/pgstac-settings.sql new file mode 100644 index 00000000..2827508a --- /dev/null +++ b/helm-chart/eoapi/initdb-data/settings/pgstac-settings.sql @@ -0,0 +1,12 @@ +-- Apply pgstac settings +DELETE FROM pgstac.pgstac_settings WHERE name = 'context'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', 'auto'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '100000'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '100000'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '1 day'); diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml index 26bb30b2..00d5d5b9 100644 --- a/helm-chart/eoapi/templates/db/configmap.yaml +++ b/helm-chart/eoapi/templates/db/configmap.yaml @@ -1,5 +1,4 @@ {{- if .Values.db.enabled }} ---- apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml index ff059efc..e82dcc6f 100644 --- a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml +++ b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml @@ -6,80 +6,70 @@ apiVersion: v1 kind: ConfigMap metadata: - name: initdb-sql-config-{{ $.Release.Name }} + name: pgstac-settings-config-{{ $.Release.Name }} annotations: helm.sh/hook: "post-install,post-upgrade" helm.sh/hook-weight: "-6" helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: - initdb.sql: | - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} - {{ $.Files.Get $path | nindent 4 }} - {{- end }} + pgstac-settings.sql: | + {{ $.Files.Get "initdb-data/settings/pgstac-settings.sql" | nindent 4 }} --- +{{- if or .Values.pgstacBootstrap.settings.loadSamples (eq .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES "true") }} apiVersion: v1 kind: ConfigMap metadata: - name: initdb-json-config-{{ $.Release.Name }} + name: initdb-sql-config-{{ $.Release.Name }} annotations: helm.sh/hook: "post-install,post-upgrade" helm.sh/hook-weight: "-6" helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} - {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} - {{- end }} + initdb.sql: | + {{- range $path, $bytes := $.Files.Glob "initdb-data/samples/*.sql" -}} + {{ $.Files.Get $path | nindent 4 }} + {{- end }} --- apiVersion: v1 kind: ConfigMap metadata: - name: pgstac-setup-config-{{ $.Release.Name }} + name: initdb-json-config-{{ $.Release.Name }} annotations: helm.sh/hook: "post-install,post-upgrade" helm.sh/hook-weight: "-6" helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: - pgstac-migrate.py: | - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.py" -}} - {{- $.Files.Get $path | nindent 4 -}} - {{- end }} + {{- range $path, $bytes := $.Files.Glob "initdb-data/samples/*.json" -}} + {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} + {{- end }} +{{- end }} +{{- end }} --- +{{- if .Values.postgrescluster.enabled }} apiVersion: v1 kind: ConfigMap metadata: - name: initdb-sh-config-{{ $.Release.Name }} + name: initdb annotations: helm.sh/hook: "post-install,post-upgrade" helm.sh/hook-weight: "-6" helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: - run-forever.sh: | - #!/bin/bash - pypgstac pgready --dsn $PGADMIN_URI - while true; do sleep 86400; done - apt-and-pip-install.sh: | - #!/bin/bash - apt update -y && apt install python3 python3-pip -y - pip install pypgstac[psycopg] - run-migrate-and-load.sh: | - #!/bin/bash - bash /opt/initdb/apt-and-pip-install.sh - # make sure crunchydata postgresql operator has seeded our secrets and we're ready to go - pypgstac pgready --dsn $PGADMIN_URI - # run migrations - python3 /opt/initdb/python-scripts/pgstac-migrate.py - - if [ "$LOAD_FIXTURES" = "true" ]; then - pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $PGADMIN_URI --method insert_ignore - pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $PGADMIN_URI --method insert_ignore - psql $PGADMIN_URI -f /opt/initdb/sql-data/initdb.sql - echo "DONE LOADING!!!!!!" - fi - - if [ "$KEEP_ALIVE" = "true" ]; then - bash /opt/initdb/run-forever.sh - fi - - # let the k8's pod know we've completed successfully - exit 0 + initdb.sql: | + \c {{ .Values.pgstacBootstrap.settings.database }} + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE EXTENSION IF NOT EXISTS btree_gist; + CREATE EXTENSION IF NOT EXISTS unaccent; + CREATE ROLE pgstac_admin; + CREATE ROLE pgstac_read; + CREATE ROLE pgstac_ingest; + ALTER DATABASE {{ .Values.pgstacBootstrap.settings.database }} OWNER TO {{ .Values.pgstacBootstrap.settings.user }}; + ALTER USER {{ .Values.pgstacBootstrap.settings.user }} SET search_path TO pgstac, public; + ALTER DATABASE {{ .Values.pgstacBootstrap.settings.database }} set search_path to pgstac, public; + GRANT CONNECT ON DATABASE {{ .Values.pgstacBootstrap.settings.database }} TO {{ .Values.pgstacBootstrap.settings.user }}; + GRANT ALL PRIVILEGES ON TABLES TO {{ .Values.pgstacBootstrap.settings.user }}; + GRANT ALL PRIVILEGES ON SEQUENCES TO {{ .Values.pgstacBootstrap.settings.user }}; + GRANT pgstac_read TO {{ .Values.pgstacBootstrap.settings.user }} WITH ADMIN OPTION; + GRANT pgstac_ingest TO {{ .Values.pgstacBootstrap.settings.user }} WITH ADMIN OPTION; + GRANT pgstac_admin TO {{ .Values.pgstacBootstrap.settings.user }} WITH ADMIN OPTION; {{- end }} diff --git a/helm-chart/eoapi/templates/pgstacboostrap/job.yaml b/helm-chart/eoapi/templates/pgstacboostrap/job.yaml index 4ebcf16b..ec8cbeb3 100644 --- a/helm-chart/eoapi/templates/pgstacboostrap/job.yaml +++ b/helm-chart/eoapi/templates/pgstacboostrap/job.yaml @@ -10,12 +10,22 @@ # 3. The "before-hook-creation,hook-succeeded" delete policy ensures the job is recreated # on each helm install/upgrade and is cleaned up after successful completion # +# DEBUGGING NOTES: +# If the job fails with "BackoffLimitExceeded" error: +# 1. Check database connectivity - ensure the database is running and accessible +# 2. Check database credentials - make sure POSTGRES_HOST, POSTGRES_PORT, etc. are correct +# 3. Check for pgstac migration errors - examine container logs for migration failures +# 4. Verify database initialization - ensure postgres extensions and roles are created +# +# The CI workflow has been enhanced with a debug step that captures comprehensive logs +# from all pgstac-migrate pods to help diagnose failures during testing. +# apiVersion: batch/v1 kind: Job metadata: - name: pgstacbootstrap + name: pgstac-migrate labels: - app: pgstacbootstrap + app: pgstac-migrate annotations: helm.sh/hook: "post-install,post-upgrade" helm.sh/hook-weight: "-5" @@ -24,33 +34,118 @@ spec: template: metadata: labels: - app: pgstacbootstrap + app: pgstac-migrate + spec: + restartPolicy: Never + containers: + - name: pgstac-migrate + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + command: + - "/bin/sh" + - "-c" + args: + - | + # Set standard PG environment variables from secrets + export PGHOST=$POSTGRES_HOST + export PGPORT=$POSTGRES_PORT + export PGUSER=$POSTGRES_USER + export PGPASSWORD=$POSTGRES_PASS + export PGDATABASE=$POSTGRES_DBNAME + + # Wait for the database to be ready + echo "Waiting for database to be ready..." + pypgstac pgready + + # Exit immediately if a command exits with a non-zero status + set -e + + # Run migration + echo "Running PgSTAC migrations..." + pypgstac migrate + + # Apply settings + echo "Applying PgSTAC settings..." + psql -f /opt/settings/pgstac-settings.sql + + echo "PgSTAC migration and setup complete" + resources: + {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} + volumeMounts: + - mountPath: /opt/settings + name: pgstac-settings-volume-{{ $.Release.Name }} + env: + {{ include "eoapi.pgstacSecrets" . | nindent 12 }} + volumes: + - name: pgstac-settings-volume-{{ $.Release.Name }} + configMap: + name: pgstac-settings-config-{{ $.Release.Name }} + {{- with .Values.pgstacBootstrap.settings.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pgstacBootstrap.settings.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + backoffLimit: 1 +{{- end }} + +{{- if and .Values.pgstacBootstrap.enabled (or .Values.pgstacBootstrap.settings.loadSamples (eq .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES "true")) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: pgstac-load-samples + labels: + app: pgstac-load-samples + annotations: + helm.sh/hook: "post-install,post-upgrade" + helm.sh/hook-weight: "-4" + helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" +spec: + template: + metadata: + labels: + app: pgstac-load-samples spec: restartPolicy: Never containers: - - name: pgstacbootstrap + - name: pgstac-load-samples image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} - command: {{- toYaml .Values.pgstacBootstrap.command | nindent 12 }} + command: + - "/bin/sh" + - "-c" args: - {{- toYaml .Values.pgstacBootstrap.args | nindent 12 }} - ports: - - containerPort: 5432 + - | + # Exit immediately if a command exits with a non-zero status + set -e + + # Set standard PG environment variables from secrets + export PGHOST=$POSTGRES_HOST + export PGPORT=$POSTGRES_PORT + export PGUSER=$POSTGRES_USER + export PGPASSWORD=$POSTGRES_PASS + export PGDATABASE=$POSTGRES_DBNAME + + # Load sample data + echo "Loading sample collections..." + pypgstac load collections /opt/data/noaa-emergency-response.json --method insert_ignore + + echo "Loading sample items..." + pypgstac load items /opt/data/noaa-eri-nashville2020.json --method insert_ignore + + echo "Running custom SQL..." + psql -f /opt/sql/initdb.sql + + echo "Sample data loading complete" resources: {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} volumeMounts: - - mountPath: /opt/initdb/sql-data + - mountPath: /opt/sql name: initdb-sql-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/json-data + - mountPath: /opt/data name: initdb-json-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/ - name: initdb-sh-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/python-scripts - name: pgstac-setup-volume-{{ $.Release.Name }} env: - - name: LOAD_FIXTURES - value: {{ .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES | quote }} - - name: KEEP_ALIVE - value: {{ .Values.pgstacBootstrap.settings.envVars.KEEP_ALIVE | quote }} {{ include "eoapi.pgstacSecrets" . | nindent 12 }} volumes: - name: initdb-sql-volume-{{ $.Release.Name }} @@ -59,12 +154,6 @@ spec: - name: initdb-json-volume-{{ $.Release.Name }} configMap: name: initdb-json-config-{{ $.Release.Name }} - - name: initdb-sh-volume-{{ $.Release.Name }} - configMap: - name: initdb-sh-config-{{ $.Release.Name }} - - name: pgstac-setup-volume-{{ $.Release.Name }} - configMap: - name: pgstac-setup-config-{{ $.Release.Name }} {{- with .Values.pgstacBootstrap.settings.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 4da68908..9a5136dd 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -105,6 +105,18 @@ postgrescluster: postGISVersion: 3.4 pgBouncerReplicas: 1 monitoring: false + # Configure Patroni to set proper schema permissions + patroni: + dynamicConfiguration: + postgresql: + pg_hba: + - "host all all 0.0.0.0/0 md5" + parameters: + shared_preload_libraries: pg_stat_statements, auto_explain + databaseInitSQL: + key: initdb.sql + name: initdb + instances: - name: eoapi replicas: 1 @@ -118,22 +130,21 @@ postgrescluster: memory: "3048Mi" # https://access.crunchydata.com/documentation/postgres-operator/latest/architecture/user-management users: - # `postgres` always is a `SUPERUSER` and cannot be adjusted with `options` flag + # `postgres` always has SUPERUSER privileges # we use this block here to create secrets for the `postgres` user that we can mount and lookup - name: postgres databases: - eoapi - postgres - # if you try to grant `options: 'SUPERUSER'` below to your eoapi service user - # then they will not be able to connect through the pgbouncer connection pooler, so be aware + options: "SUPERUSER" + # Grant CREATEROLE privilege to eoapi user so it can create the pgstac_admin role + # If you grant SUPERUSER to the eoapi user, it won't be able to connect through pgbouncer # https://access.crunchydata.com/documentation/crunchy-postgres-containers/2.4.2/container-specifications/crunchy-pgbouncer/ - # this user is currently unprivileged and the `pgstacBootstrap` container below will grant privileges - # when it boots via the `postgres` superuser above. - # see `/helm-chart/eoapi/templates/pgstacbootstrap/configmap.yaml:initdb-sh-config` - name: eoapi databases: - eoapi - postgres + options: "CREATEDB CREATEROLE" # default `password.type` is ASCII which follows the character set US-ASCII # but which can contain characters that `asyncpg` dislikes # see https://github.com/MagicStack/asyncpg/issues/1151 @@ -146,13 +157,15 @@ postgrescluster: pgstacBootstrap: enabled: true image: - name: ghcr.io/stac-utils/pgstac + name: ghcr.io/stac-utils/pgstac-pypgstac tag: v0.9.5 - command: - - "sh" - args: - - "/opt/initdb/run-migrate-and-load.sh" settings: + # General configuration options + loadSamples: true # Set to false to disable sample data loading + ### Database connection settings TEMPORARY UNTIL WE HAVE A BETTER SOLUTION + ### FOR CONFIGURING THE DB CONNECTION + user: eoapi + database: eoapi resources: requests: cpu: "512m" @@ -160,11 +173,10 @@ pgstacBootstrap: limits: cpu: "512m" memory: "1024Mi" + # DEPRECATED: Use pgstacBootstrap.settings.loadSamples instead envVars: # toggle to "false" if you don't want fixtures default loaded LOAD_FIXTURES: "true" - # toggle to `true` if you want to keep the job running as db jumpbox - KEEP_ALIVE: "false" ######################