Merge pull request #81 from DataDog/TRAIN-2439-postgres-logs

Fixes postgres service logging

Author: arosenkranz

docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - /proc/:/host/proc/:ro
       - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+      - postgres_logs:/var/log/pg_log:ro
   frontend:
     build:
       context: ./services/frontend
@@ -101,12 +102,15 @@ services:
     restart: always
     depends_on:
       - dd-agent
+    volumes:
+      # save postgres logs to a volume for datadog to collect
+      - postgres_logs:/var/log/pg_log:rw
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
       - POSTGRES_USER
       - POSTGRES_PASSWORD
       - DD_ENV=${DD_ENV-dev}
-      - DD_SERVICE=storedog-db
+      - DD_SERVICE=postgres
       - DD_VERSION=${DD_VERSION-7}
       - DD_AGENT_HOST=dd-agent
       - DD_DBM_PROPAGATION_MODE=full
@@ -115,7 +119,7 @@ services:
       - DD_PROFILING_ENABLED=true
     labels:
       com.datadoghq.tags.env: '${DD_ENV}'
-      com.datadoghq.tags.service: 'storedog-db'
+      com.datadoghq.tags.service: 'postgres'
       com.datadoghq.tags.version: '13'
       com.datadoghq.ad.check_names: '["postgres", "postgres"]'
       com.datadoghq.ad.init_configs: '[{}, {}]'
@@ -165,7 +169,7 @@ services:
         "enabled": true
         }
         }]'
-      com.datadoghq.ad.logs: '[{"source": "postgresql", "service": "postgres", "auto_multi_line_detection":true }]'
+      com.datadoghq.ad.logs: '[{"source": "postgresql", "service": "postgres", "auto_multi_line_detection":true, "path": "/var/log/pg_log/postgresql*.json", "type": "file"}]'
       my.custom.label.team: 'database'
   redis:
     image: redis:6.2-alpine
@@ -375,3 +379,4 @@ services:
 
 volumes:
   redis:
+  postgres_logs:

services/postgres/Dockerfile

@@ -1,3 +1,26 @@
-FROM postgres:13-alpine
+FROM postgres:15-alpine
+
+# Install dcron and su-exec for user switching
+RUN apk update && apk add --no-cache dcron su-exec
+
+# Copy SQL dump and PostgreSQL configuration
 COPY ./scripts/restore-2024-04-25-22-06-17.sql /docker-entrypoint-initdb.d/
-COPY ./postgresql.conf /postgresql.conf
+COPY ./scripts/postgresql.conf /postgresql.conf
+
+# Create log directory and set permissions
+RUN mkdir -p /var/log/pg_log && \
+  chown -R postgres:postgres /var/log/pg_log && \
+  chmod -R 755 /var/log/pg_log
+
+# Copy the cron job file
+COPY ./scripts/cleanup-cron /etc/crontabs/root
+
+# Copy and set the entrypoint script
+COPY ./scripts/start-services.sh /usr/local/bin/start-services.sh
+RUN chmod +x /usr/local/bin/start-services.sh
+
+# Use the custom entrypoint script
+ENTRYPOINT ["/usr/local/bin/start-services.sh"]
+
+# Default command
+CMD ["postgres", "-c", "config_file=/postgresql.conf"]

services/postgres/scripts/cleanup-cron

@@ -0,0 +1 @@
+*/1 * * * * /usr/bin/find /var/log/pg_log/ -type f -name "*" -mmin +5 -delete >> /var/log/cron.log 2>&1

services/postgres/postgresql.conf renamed to services/postgres/scripts/postgresql.conf

@@ -423,35 +423,35 @@ min_wal_size = 80MB
 
 # - Where to Log -
 
-log_destination = 'stderr'             # Valid values are combinations of
+log_destination = 'jsonlog'             # Valid values are combinations of
                                         # stderr, csvlog, syslog, and eventlog,
                                         # depending on platform.  csvlog
                                         # requires logging_collector to be on.
 
 # This is used when logging to stderr:
-# logging_collector = on                  # Enable capturing of stderr and csvlog
+logging_collector = on                   # Enable capturing of stderr and  csvlog
                                         # into log files. Required to be on for
                                         # csvlogs.
                                         # (change requires restart)
 
 # These are only used if logging_collector is on:
-log_directory = 'pg_log'                  # directory where log files are written,
+log_directory = '/var/log/pg_log'                  # directory where log files are written,
                                         # can be absolute or relative to PGDATA
-log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'        # log file name pattern,
+log_filename = 'postgresql-%Y-%m-%d_%H-%M'  # log file name pattern,
                                         # can include strftime() escapes
 log_file_mode = 0644                   # creation mode for log files,
                                         # begin with 0 to use octal notation
-#log_truncate_on_rotation = off         # If on, an existing log file with the
+log_truncate_on_rotation = on         # If on, an existing log file with the
                                         # same name as the new log file will be
                                         # truncated rather than appended to.
                                         # But such truncation only occurs on
                                         # time-driven rotation, not on restarts
                                         # or size-driven rotation.  Default is
                                         # off, meaning append to existing files
                                         # in all cases.
-#log_rotation_age = 1d                  # Automatic rotation of logfiles will
+log_rotation_age = '4min'                  # Automatic rotation of logfiles will
                                         # happen after that time.  0 disables.
-#log_rotation_size = 10MB               # Automatic rotation of logfiles will
+log_rotation_size = 0               # Automatic rotation of logfiles will
                                         # happen after that much log output.
                                         # 0 disables.
 
@@ -524,7 +524,7 @@ log_min_duration_statement = 0       # -1 is disabled, 0 logs all statements
 #log_checkpoints = off
 #log_connections = off
 #log_disconnections = off
-log_duration = on
+# log_duration = on
 #log_error_verbosity = default          # terse, default, or verbose messages
 #log_hostname = off
 log_line_prefix= '%m [%p] %d %a %u %h %c '           # special values:
@@ -556,7 +556,7 @@ log_line_prefix= '%m [%p] %d %a %u %h %c '           # special values:
 #log_parameter_max_length_on_error = 0  # when logging an error, limit logged
                                         # bind-parameter values to N bytes;
                                         # -1 means print in full, 0 disables
-log_statement = 'all'                   # none, ddl, mod, all
+#log_statement = 'all'                   # none, ddl, mod, all
 #log_replication_commands = off
 #log_temp_files = -1                    # log temporary files equal or larger
                                         # than the specified size in kilobytes;

services/postgres/scripts/start-services.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Start cron as root
+crond -f &
+
+# Ensure data directory has correct permissions
+chown -R postgres:postgres /var/lib/postgresql/data
+chmod -R 0700 /var/lib/postgresql/data
+
+# Switch to the postgres user and start PostgreSQL using the original entrypoint
+exec su-exec postgres docker-entrypoint.sh "$@"