From 29f11642e15f314fc72c915dcd66cf6c00db1d9a Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 1 Sep 2023 14:20:02 -0500 Subject: [PATCH 01/25] Add cgmap container --- chartpress.yaml | 4 ++- compose/cgmap.yml | 19 +++++++++++ images/cgmap/Dockerfile | 47 +++++++++++++++++++++++++ images/cgmap/README.md | 22 ++++++++++++ images/cgmap/lighttpd.conf | 70 ++++++++++++++++++++++++++++++++++++++ images/cgmap/start.sh | 13 +++++++ osm-seed/values.yaml | 19 +++++++++++ 7 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 compose/cgmap.yml create mode 100644 images/cgmap/Dockerfile create mode 100644 images/cgmap/README.md create mode 100644 images/cgmap/lighttpd.conf create mode 100755 images/cgmap/start.sh diff --git a/chartpress.yaml b/chartpress.yaml index 98f0f471..fcbbac9f 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -41,4 +41,6 @@ charts: osm-simple-metrics: valuesPath: osmSimpleMetrics.image changeset-replication-job: - valuesPath: changesetReplicationJob.image \ No newline at end of file + valuesPath: changesetReplicationJob.image + cgmap: + valuesPath: cgmap.image \ No newline at end of file diff --git a/compose/cgmap.yml b/compose/cgmap.yml new file mode 100644 index 00000000..6ac51c90 --- /dev/null +++ b/compose/cgmap.yml @@ -0,0 +1,19 @@ +version: '3' +services: + # ##################################################### + # ## cgmap section + # ##################################################### + cgmap: + image: osmseed-cgmap:v1 + build: + context: ../images/cgmap + dockerfile: Dockerfile + ports: + # - '8000:8000' + - '80:80' + volumes: + - ../data/cgmap-data:/apps/data/ + # command: > + # /bin/bash -c " ./start.sh" + env_file: + - ../envs/.env.db diff --git a/images/cgmap/Dockerfile b/images/cgmap/Dockerfile new file mode 100644 index 00000000..bf78d66f --- /dev/null +++ b/images/cgmap/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:20.04 AS builder + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -qq && \ + apt-get install -y gcc g++ make autoconf automake libtool \ + libfcgi-dev libxml2-dev libmemcached-dev \ + libboost-program-options-dev \ + libcrypto++-dev libyajl-dev \ + libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \ + postgresql-12 postgresql-server-dev-all \ + git ca-certificates \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git ./ +RUN git checkout v0.8.8 + +# Compile, install and remove source +RUN ./autogen.sh && \ + ./configure --enable-static --disable-shared --enable-yajl && \ + make -j3 && \ + make check && \ + strip openstreetmap-cgimap + +FROM ubuntu:20.04 + +RUN apt-get update -qq && \ + apt-get install -y \ + libfcgi-bin libmemcached11 libboost-locale1.71.0 libboost-program-options1.71.0 \ + libxml2 libcrypto++6 libyajl2 libpqxx-6.4 zlib1g libargon2-1 \ + --no-install-recommends lighttpd && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/openstreetmap-cgimap /usr/local/bin + +RUN groupadd -g 61000 cgimap && \ + useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap + +USER cgimap +COPY start.sh / +COPY lighttpd.conf / +CMD /start.sh \ No newline at end of file diff --git a/images/cgmap/README.md b/images/cgmap/README.md new file mode 100644 index 00000000..22968868 --- /dev/null +++ b/images/cgmap/README.md @@ -0,0 +1,22 @@ +# openstreetmap-cgimap + +This container is built using the configuration from Zerebubuth's OpenStreetMap CGImap GitHub repository, with minor modifications. + + +# Build and up + +```sh +docker compose -f compose/cgmap.yml build +docker compose -f compose/cgmap.yml up +``` + +Note: Ensure that you are running PostgreSQL on your local machine. For example: + + +```sh +kubectl port-forward staging-db-0 5432:5432 +``` + +Check results: + +http://localhost/api/0.6/map?bbox=-77.09529161453248,-12.071898885565846,-77.077374458313,-12.066474684936727 diff --git a/images/cgmap/lighttpd.conf b/images/cgmap/lighttpd.conf new file mode 100644 index 00000000..7e42430e --- /dev/null +++ b/images/cgmap/lighttpd.conf @@ -0,0 +1,70 @@ +# default document-root +server.document-root = "/openstreetmap/cgimap/" + +# TCP port +server.port = 80 + +server.reject-expect-100-with-417 = "disable" + +# selecting modules +server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias" ) + +# handling unknown routes +server.error-handler-404 = "/dispatch.map" + +# include, relative to dirname of main config file +#include "mime.types.conf" + +# read configuration from output of a command +#include_shell "/usr/local/bin/confmimetype /etc/mime.types" +mimetype.assign = ( + ".html" => "text/html", + ".txt" => "text/plain", + ".jpg" => "image/jpeg", + ".png" => "image/png" +) + +#debug.log-request-handling = "enable" + +$HTTP["request-method"] == "GET" { + url.rewrite-once = ( + "^/api/0\.6/map(\.(json|xml))?(\?(.*))?$" => "/dispatch.map", + "^/api/0\.6/(node|way|relation)/[[:digit:]]+(\.(json|xml))?$" => "/dispatch.map", + "^/api/0\.6/(node|way|relation)/[[:digit:]]+/history.*$" => "/dispatch.map", + "^/api/0\.6/(node|way|relation)/[[:digit:]]+/[[:digit:]]+.*$" => "/dispatch.map", + "^/api/0\.6/(node|way|relation)/[[:digit:]]+/relations$" => "/dispatch.map", + "^/api/0\.6/node/[[:digit:]]+/ways$" => "/dispatch.map", + "^/api/0\.6/(way|relation)/[[:digit:]]+/full$" => "/dispatch.map", + "^/api/0\.6/changeset/[[:digit:]]+.*$" => "/dispatch.map", + "^/api/0\.6/(nodes|ways|relations)(\?(.*))?$" => "/dispatch.map", + "^/api/0\.6/changeset/[[:digit:]]+/download$" => "/dispatch.map", + ) +} + +$HTTP["request-method"] == "POST" { + url.rewrite-once = ( + "^/api/0\.6/changeset/[[:digit:]]+/upload.*$" => "/dispatch.map", + ) +} + +$HTTP["request-method"] == "PUT" { + url.rewrite-once = ( + "^/api/0\.6/changeset/[[:digit:]]+/close.*$" => "/dispatch.map", + "^/api/0\.6/changeset/[[:digit:]]+$" => "/dispatch.map", + "^/api/0\.6/changeset/create.*$" => "/dispatch.map", + ) +} + +$HTTP["url"] =~ "^/(?!(dispatch\.map))" { + proxy.server = ( "" => ( ( "host" => "127.0.0.1", "port" => "3000" ) ) ) +} + +fastcgi.debug = 1 + +fastcgi.server = ( ".map" => + (( "host" => "127.0.0.1", + "port" => 8000, + "check-local" => "disable", + )) +) + diff --git a/images/cgmap/start.sh b/images/cgmap/start.sh new file mode 100755 index 00000000..cabca133 --- /dev/null +++ b/images/cgmap/start.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -ex + +/usr/sbin/lighttpd -f lighttpd.conf + +/usr/local/bin/openstreetmap-cgimap \ + --port=8000 \ + --instances=30 \ + --dbname=$POSTGRES_DB \ + --host=$POSTGRES_HOST \ + --username=$POSTGRES_USER \ + --password=$POSTGRES_PASSWORD + diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index b9f45f6c..89e428ac 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -716,3 +716,22 @@ changesetReplicationJob: cpu: '10' nodeSelector: enabled: false + +# ==================================================================================================== +# Variables for cgmap config +# ==================================================================================================== +cgmap: + enabled: false + image: + name: '' + tag: '' + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false From e0f2388616d1abb2bf985b38ad0f09fd87c22c9d Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 1 Sep 2023 14:26:10 -0500 Subject: [PATCH 02/25] Correct the misspelled name --- compose/{cgmap.yml => cgimap.yml} | 8 ++++---- images/{cgmap => cgimap}/Dockerfile | 0 images/{cgmap => cgimap}/README.md | 4 ++-- images/{cgmap => cgimap}/lighttpd.conf | 0 images/{cgmap => cgimap}/start.sh | 0 osm-seed/values.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename compose/{cgmap.yml => cgimap.yml} (76%) rename images/{cgmap => cgimap}/Dockerfile (100%) rename images/{cgmap => cgimap}/README.md (84%) rename images/{cgmap => cgimap}/lighttpd.conf (100%) rename images/{cgmap => cgimap}/start.sh (100%) diff --git a/compose/cgmap.yml b/compose/cgimap.yml similarity index 76% rename from compose/cgmap.yml rename to compose/cgimap.yml index 6ac51c90..099cbd26 100644 --- a/compose/cgmap.yml +++ b/compose/cgimap.yml @@ -3,16 +3,16 @@ services: # ##################################################### # ## cgmap section # ##################################################### - cgmap: - image: osmseed-cgmap:v1 + cgimap: + image: osmseed-cgimap:v1 build: - context: ../images/cgmap + context: ../images/cgimap dockerfile: Dockerfile ports: # - '8000:8000' - '80:80' volumes: - - ../data/cgmap-data:/apps/data/ + - ../data/cgimap-data:/apps/data/ # command: > # /bin/bash -c " ./start.sh" env_file: diff --git a/images/cgmap/Dockerfile b/images/cgimap/Dockerfile similarity index 100% rename from images/cgmap/Dockerfile rename to images/cgimap/Dockerfile diff --git a/images/cgmap/README.md b/images/cgimap/README.md similarity index 84% rename from images/cgmap/README.md rename to images/cgimap/README.md index 22968868..92972a25 100644 --- a/images/cgmap/README.md +++ b/images/cgimap/README.md @@ -6,8 +6,8 @@ This container is built using the configuration from Zerebubuth's OpenStreetMap # Build and up ```sh -docker compose -f compose/cgmap.yml build -docker compose -f compose/cgmap.yml up +docker compose -f compose/cgimap.yml build +docker compose -f compose/cgimap.yml up ``` Note: Ensure that you are running PostgreSQL on your local machine. For example: diff --git a/images/cgmap/lighttpd.conf b/images/cgimap/lighttpd.conf similarity index 100% rename from images/cgmap/lighttpd.conf rename to images/cgimap/lighttpd.conf diff --git a/images/cgmap/start.sh b/images/cgimap/start.sh similarity index 100% rename from images/cgmap/start.sh rename to images/cgimap/start.sh diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 89e428ac..41196f1c 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -720,7 +720,7 @@ changesetReplicationJob: # ==================================================================================================== # Variables for cgmap config # ==================================================================================================== -cgmap: +cgimap: enabled: false image: name: '' From a798afdbc193eb0efe334db31ed43a8c403cabe1 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 1 Sep 2023 14:54:32 -0500 Subject: [PATCH 03/25] Add helm template for cgimap --- chartpress.yaml | 4 +- compose/cgimap.yml | 1 - images/cgimap/Dockerfile | 4 +- .../{lighttpd.conf => lighttpd.conf.template} | 2 +- images/cgimap/start.sh | 4 ++ .../templates/cgimap/cgimap-deployment.yaml | 64 +++++++++++++++++++ osm-seed/templates/cgimap/cgimap-ingress.yaml | 26 ++++++++ osm-seed/templates/cgimap/cgimap-service.yaml | 43 +++++++++++++ 8 files changed, 142 insertions(+), 6 deletions(-) rename images/cgimap/{lighttpd.conf => lighttpd.conf.template} (95%) create mode 100644 osm-seed/templates/cgimap/cgimap-deployment.yaml create mode 100644 osm-seed/templates/cgimap/cgimap-ingress.yaml create mode 100644 osm-seed/templates/cgimap/cgimap-service.yaml diff --git a/chartpress.yaml b/chartpress.yaml index fcbbac9f..79aeab27 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -42,5 +42,5 @@ charts: valuesPath: osmSimpleMetrics.image changeset-replication-job: valuesPath: changesetReplicationJob.image - cgmap: - valuesPath: cgmap.image \ No newline at end of file + cgimap: + valuesPath: cgimap.image \ No newline at end of file diff --git a/compose/cgimap.yml b/compose/cgimap.yml index 099cbd26..97c0cf42 100644 --- a/compose/cgimap.yml +++ b/compose/cgimap.yml @@ -9,7 +9,6 @@ services: context: ../images/cgimap dockerfile: Dockerfile ports: - # - '8000:8000' - '80:80' volumes: - ../data/cgimap-data:/apps/data/ diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index bf78d66f..916fdb35 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -32,7 +32,7 @@ RUN apt-get update -qq && \ apt-get install -y \ libfcgi-bin libmemcached11 libboost-locale1.71.0 libboost-program-options1.71.0 \ libxml2 libcrypto++6 libyajl2 libpqxx-6.4 zlib1g libargon2-1 \ - --no-install-recommends lighttpd && \ + --no-install-recommends lighttpd gettext-base && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -43,5 +43,5 @@ RUN groupadd -g 61000 cgimap && \ USER cgimap COPY start.sh / -COPY lighttpd.conf / +COPY lighttpd.conf.template / CMD /start.sh \ No newline at end of file diff --git a/images/cgimap/lighttpd.conf b/images/cgimap/lighttpd.conf.template similarity index 95% rename from images/cgimap/lighttpd.conf rename to images/cgimap/lighttpd.conf.template index 7e42430e..b2724f73 100644 --- a/images/cgimap/lighttpd.conf +++ b/images/cgimap/lighttpd.conf.template @@ -56,7 +56,7 @@ $HTTP["request-method"] == "PUT" { } $HTTP["url"] =~ "^/(?!(dispatch\.map))" { - proxy.server = ( "" => ( ( "host" => "127.0.0.1", "port" => "3000" ) ) ) + proxy.server = ( "" => ( ( "host" => "${API_WEB_HOST}", "port" => "${API_WEB_PORT}" ) ) ) } fastcgi.debug = 1 diff --git a/images/cgimap/start.sh b/images/cgimap/start.sh index cabca133..c10a7566 100755 --- a/images/cgimap/start.sh +++ b/images/cgimap/start.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -ex +# Make sure that the follow env vars has been declare +# API_WEB_HOST +# API_WEB_PORT +envsubst < lighttpd.conf.template > lighttpd.conf /usr/sbin/lighttpd -f lighttpd.conf /usr/local/bin/openstreetmap-cgimap \ diff --git a/osm-seed/templates/cgimap/cgimap-deployment.yaml b/osm-seed/templates/cgimap/cgimap-deployment.yaml new file mode 100644 index 00000000..91dd7b31 --- /dev/null +++ b/osm-seed/templates/cgimap/cgimap-deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.cgimap.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "osm-seed.fullname" . }}-cgimap + labels: + app: {{ template "osm-seed.name" . }} + component: cgimap-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.cgimap.replicaCount }} + selector: + matchLabels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-cgimap + template: + metadata: + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-cgimap + spec: + containers: + - name: {{ .Chart.Name }}-cgimap + image: "{{ .Values.cgimap.image.name }}:{{ .Values.cgimap.image.tag }}" + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 600 # 10 min, because the compile process takes time. + timeoutSeconds: 30 + {{- if .Values.cgimap.resources.enabled }} + resources: + requests: + memory: {{ .Values.cgimap.resources.requests.memory }} + cpu: {{ .Values.cgimap.resources.requests.cpu }} + limits: + memory: {{ .Values.cgimap.resources.limits.memory }} + cpu: {{ .Values.cgimap.resources.limits.cpu }} + {{- end }} + env: + - name: POSTGRES_HOST + value: {{ .Release.Name }}-db + - name: POSTGRES_DB + value: {{ .Values.db.env.POSTGRES_DB }} + - name: POSTGRES_PASSWORD + value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} + - name: POSTGRES_USER + value: {{ .Values.db.env.POSTGRES_USER }} + - name: API_WEB_HOST + value: {{ .Release.Name }}-web + - name: API_WEB_PORT + value: 80 + {{- if .Values.cgimap.nodeSelector.enabled }} + nodeSelector: + {{ .Values.cgimap.nodeSelector.label_key }} : {{ .Values.cgimap.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/osm-seed/templates/cgimap/cgimap-ingress.yaml b/osm-seed/templates/cgimap/cgimap-ingress.yaml new file mode 100644 index 00000000..7c158b60 --- /dev/null +++ b/osm-seed/templates/cgimap/cgimap-ingress.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.cgimap.enabled (eq .Values.serviceType "ClusterIP") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "osm-seed.fullname" . }}-ingress-cgimap-api + annotations: + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer +spec: + tls: + - hosts: + - cgimap.{{ .Values.domain }} + secretName: {{ template "osm-seed.fullname" . }}-secret-cgimap + + rules: + - host: cgimap.{{ .Values.domain }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ template "osm-seed.fullname" . }}-cgimap + port: + number: 80 +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/cgimap/cgimap-service.yaml b/osm-seed/templates/cgimap/cgimap-service.yaml new file mode 100644 index 00000000..a7449407 --- /dev/null +++ b/osm-seed/templates/cgimap/cgimap-service.yaml @@ -0,0 +1,43 @@ +{{- if .Values.cgimap.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "osm-seed.fullname" . }}-cgimap + labels: + app: {{ template "osm-seed.name" . }} + component: cgimap-service + environment: {{ .Values.environment }} + release: {{ .Release.Name }} + annotations: + {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + {{- end }} + {{- if eq .Values.serviceType "ClusterIP" }} + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer + {{- else }} + fake.annotation: fake + {{- end }} + {{- with .Values.cgimap.serviceAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.serviceType }} + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + - port: 443 + targetPort: http + protocol: TCP + name: https + {{- end }} + selector: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-cgimap +{{- end }} \ No newline at end of file From 9a90c8a188ac1e4371272103441127f9d39a9971 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 20 Jun 2024 13:17:43 -0500 Subject: [PATCH 04/25] Update cgimap dockerfile --- images/cgimap/Dockerfile | 65 ++++++++++++++++++++++++++-------------- images/cgimap/start.sh | 43 ++++++++++++++++---------- 2 files changed, 70 insertions(+), 38 deletions(-) diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index 916fdb35..85ae3231 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -1,38 +1,53 @@ -FROM ubuntu:20.04 AS builder +FROM ubuntu:24.04 AS builder ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -qq && \ - apt-get install -y gcc g++ make autoconf automake libtool \ - libfcgi-dev libxml2-dev libmemcached-dev \ - libboost-program-options-dev \ - libcrypto++-dev libyajl-dev \ - libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \ - postgresql-12 postgresql-server-dev-all \ - git ca-certificates \ - --no-install-recommends && \ + apt-get install -y gcc g++ make cmake \ + libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \ + libboost-program-options-dev libcrypto++-dev libyajl-dev \ + libpqxx-dev zlib1g-dev libfmt-dev git \ + ca-certificates \ + postgresql-16 postgresql-server-dev-all \ + --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + + RUN apt-get update -qq && \ + apt-get install -y wget + + # Install Argon2 from source +RUN wget https://github.com/P-H-C/phc-winner-argon2/archive/refs/tags/20190702.tar.gz && \ +tar -xzf 20190702.tar.gz && \ +cd phc-winner-argon2-20190702 && \ +make && \ +make install + WORKDIR /app -RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git ./ -RUN git checkout v0.8.8 +# Copy the main application. + +ENV CGIMAP_GITSHA=26cd7fa10affe5dbd13dbe16de34421059f53f18 +RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app +RUN git checkout $CGIMAP_GITSHA + # Compile, install and remove source -RUN ./autogen.sh && \ - ./configure --enable-static --disable-shared --enable-yajl && \ - make -j3 && \ - make check && \ - strip openstreetmap-cgimap +RUN mkdir build && cd build && \ + CXXFLAGS="-Wall -Wextra -Wpedantic -Wno-unused-parameter" cmake .. -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release && \ + make -j${nproc} && \ + ctest --output-on-failure && \ + strip openstreetmap-cgimap && \ + cp openstreetmap-cgimap ../ -FROM ubuntu:20.04 +FROM ubuntu:24.04 RUN apt-get update -qq && \ apt-get install -y \ - libfcgi-bin libmemcached11 libboost-locale1.71.0 libboost-program-options1.71.0 \ - libxml2 libcrypto++6 libyajl2 libpqxx-6.4 zlib1g libargon2-1 \ - --no-install-recommends lighttpd gettext-base && \ + libfcgi-bin libmemcached11 libboost-program-options1.83.0 \ + libxml2 libcrypto++8 libyajl2 libpqxx-7.8t64 zlib1g libbrotli1 libfmt9 \ + --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -42,6 +57,10 @@ RUN groupadd -g 61000 cgimap && \ useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap USER cgimap -COPY start.sh / -COPY lighttpd.conf.template / -CMD /start.sh \ No newline at end of file + +COPY start.sh / + +EXPOSE 8000 + +# ENTRYPOINT +CMD [ "/start.sh" ] \ No newline at end of file diff --git a/images/cgimap/start.sh b/images/cgimap/start.sh index c10a7566..7db92bbe 100755 --- a/images/cgimap/start.sh +++ b/images/cgimap/start.sh @@ -1,17 +1,30 @@ #!/usr/bin/env bash -set -ex - -# Make sure that the follow env vars has been declare -# API_WEB_HOST -# API_WEB_PORT -envsubst < lighttpd.conf.template > lighttpd.conf -/usr/sbin/lighttpd -f lighttpd.conf - -/usr/local/bin/openstreetmap-cgimap \ - --port=8000 \ - --instances=30 \ - --dbname=$POSTGRES_DB \ - --host=$POSTGRES_HOST \ - --username=$POSTGRES_USER \ - --password=$POSTGRES_PASSWORD +export CGIMAP_HOST=$POSTGRES_HOST +export CGIMAP_DBNAME=$POSTGRES_DB +export CGIMAP_USERNAME=$POSTGRES_USER +export CGIMAP_PASSWORD=$POSTGRES_PASSWORD +export CGIMAP_OAUTH_HOST=$POSTGRES_HOST +export CGIMAP_UPDATE_HOST=$POSTGRES_HOST +export CGIMAP_LOGFILE="/var/www/log/cgimap.log" +export CGIMAP_MEMCACHE=$OPENSTREETMAP_memcache_servers +export CGIMAP_RATELIMIT="204800" +export CGIMAP_MAXDEBT="250" +export CGIMAP_MAP_AREA="0.25" +export CGIMAP_MAP_NODES="100000" +export CGIMAP_MAX_WAY_NODES="2000" +export CGIMAP_MAX_RELATION_MEMBERS="32000" +# export CGIMAP_RATELIMIT_UPLOAD="true" +# export CGIMAP_MODERATOR_RATELIMIT="1048576" +# export CGIMAP_MODERATOR_MAXDEBT="1280" +# export CGIMAP_PIDFILE="/var/www/cgimap.pid" +# Check for web site status +if [[ "$WEBSITE_STATUS" == "database_readonly" || "$WEBSITE_STATUS" == "api_readonly" ]]; then + export CGIMAP_DISABLE_API_WRITE="true" +fi +if [[ "$WEBSITE_STATUS" == "database_offline" || "$WEBSITE_STATUS" == "api_offline" ]]; then + echo "Website is $WEBSITE_STATUS. No action required for cgimap service." +else + /usr/local/bin/openstreetmap-cgimap --port=8000 --daemon --instances=10 && \ + tail --pid=$(cat /tmp/cgimap.pid) -f /dev/null +fi From 951463da13defb14bd393264fb0a12e43276b2a0 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 20 Jun 2024 13:28:01 -0500 Subject: [PATCH 05/25] Update tiler-imposm dockerfile --- images/tiler-imposm/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile index e99ccc28..a996721d 100644 --- a/images/tiler-imposm/Dockerfile +++ b/images/tiler-imposm/Dockerfile @@ -23,7 +23,9 @@ RUN apt-get install -y \ rm -rf /var/lib/apt/lists/* # # Install python -RUN add-apt-repository ppa:deadsnakes/ppa && \ +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \ rm -rf /var/lib/apt/lists/* && \ From 36a3ae6c73f3be9c10de50c53094911f40018281 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 20 Jun 2024 14:50:41 -0500 Subject: [PATCH 06/25] Update python --- images/tiler-imposm/Dockerfile | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile index a996721d..19a44cfd 100644 --- a/images/tiler-imposm/Dockerfile +++ b/images/tiler-imposm/Dockerfile @@ -22,15 +22,26 @@ RUN apt-get install -y \ software-properties-common && \ rm -rf /var/lib/apt/lists/* -# # Install python +# Install Python 3.6 RUN apt-get update && \ apt-get install -y software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ apt-get update && \ - apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \ - rm -rf /var/lib/apt/lists/* && \ - python3 -m pip install pip --upgrade && \ - python3 -m pip install wheel + apt-get install -y gcc-8 g++-8 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 && \ + apt-get install -y build-essential wget libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev && \ + wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz && \ + tar xzf Python-3.6.15.tgz && \ + cd Python-3.6.15 && \ + ./configure --enable-optimizations && \ + make altinstall && \ + cd .. && \ + rm -rf Python-3.6.15 Python-3.6.15.tgz && \ + apt-get install -y python3-pip && \ + python3.6 -m pip install pip --upgrade && \ + python3.6 -m pip install wheel && \ + rm -rf /var/lib/apt/lists/* # Install postgresql-client RUN apt-get update && apt-get install -y postgresql-client && \ From 057b8a206663e4c9de5b60799b8dd96ce9265327 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 20 Jun 2024 15:34:41 -0500 Subject: [PATCH 07/25] Update cgimap container --- images/cgimap/Dockerfile | 43 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index 85ae3231..a1e72c5d 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -9,37 +9,34 @@ RUN apt-get update -qq && \ libpqxx-dev zlib1g-dev libfmt-dev git \ ca-certificates \ postgresql-16 postgresql-server-dev-all \ + wget \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* - - RUN apt-get update -qq && \ - apt-get install -y wget - - # Install Argon2 from source +# Install Argon2 from source RUN wget https://github.com/P-H-C/phc-winner-argon2/archive/refs/tags/20190702.tar.gz && \ -tar -xzf 20190702.tar.gz && \ -cd phc-winner-argon2-20190702 && \ -make && \ -make install + tar -xzf 20190702.tar.gz && \ + cd phc-winner-argon2-20190702 && \ + make && \ + make install && \ + ldconfig WORKDIR /app # Copy the main application. - ENV CGIMAP_GITSHA=26cd7fa10affe5dbd13dbe16de34421059f53f18 -RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app -RUN git checkout $CGIMAP_GITSHA - +RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app && \ + cd /app && \ + git checkout $CGIMAP_GITSHA # Compile, install and remove source RUN mkdir build && cd build && \ CXXFLAGS="-Wall -Wextra -Wpedantic -Wno-unused-parameter" cmake .. -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release && \ - make -j${nproc} && \ + make -j$(nproc) && \ ctest --output-on-failure && \ strip openstreetmap-cgimap && \ - cp openstreetmap-cgimap ../ + cp openstreetmap-cgimap /usr/local/bin FROM ubuntu:24.04 @@ -47,20 +44,28 @@ RUN apt-get update -qq && \ apt-get install -y \ libfcgi-bin libmemcached11 libboost-program-options1.83.0 \ libxml2 libcrypto++8 libyajl2 libpqxx-7.8t64 zlib1g libbrotli1 libfmt9 \ + wget \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/openstreetmap-cgimap /usr/local/bin +# Install Argon2 runtime library +RUN wget https://github.com/P-H-C/phc-winner-argon2/archive/refs/tags/20190702.tar.gz && \ + tar -xzf 20190702.tar.gz && \ + cd phc-winner-argon2-20190702 && \ + make && \ + make install && \ + ldconfig + +COPY --from=builder /usr/local/bin/openstreetmap-cgimap /usr/local/bin RUN groupadd -g 61000 cgimap && \ useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap USER cgimap -COPY start.sh / +COPY start.sh / EXPOSE 8000 -# ENTRYPOINT -CMD [ "/start.sh" ] \ No newline at end of file +CMD ["/start.sh"] \ No newline at end of file From 26c3a8059c87bcccdef9fc2f32c46fa90035ab57 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 20 Jun 2024 15:38:46 -0500 Subject: [PATCH 08/25] Update cgimap container --- images/cgimap/Dockerfile | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index a1e72c5d..0a26d642 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -37,32 +37,8 @@ RUN mkdir build && cd build && \ ctest --output-on-failure && \ strip openstreetmap-cgimap && \ cp openstreetmap-cgimap /usr/local/bin - -FROM ubuntu:24.04 - -RUN apt-get update -qq && \ - apt-get install -y \ - libfcgi-bin libmemcached11 libboost-program-options1.83.0 \ - libxml2 libcrypto++8 libyajl2 libpqxx-7.8t64 zlib1g libbrotli1 libfmt9 \ - wget \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install Argon2 runtime library -RUN wget https://github.com/P-H-C/phc-winner-argon2/archive/refs/tags/20190702.tar.gz && \ - tar -xzf 20190702.tar.gz && \ - cd phc-winner-argon2-20190702 && \ - make && \ - make install && \ - ldconfig - -COPY --from=builder /usr/local/bin/openstreetmap-cgimap /usr/local/bin - -RUN groupadd -g 61000 cgimap && \ - useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap - -USER cgimap + +RUN rm -rf /app/.git COPY start.sh / From 905e88e6cd832ab7e892262f9b943983a035ee1b Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 9 Apr 2025 14:30:06 -0500 Subject: [PATCH 09/25] Update container for cgimap --- images/cgimap/Dockerfile | 71 +++++++++++----------------- images/cgimap/lighttpd.conf.template | 70 --------------------------- images/cgimap/start.sh | 27 +++++++---- 3 files changed, 45 insertions(+), 123 deletions(-) delete mode 100644 images/cgimap/lighttpd.conf.template diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index 0a26d642..1d438d4a 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -1,47 +1,30 @@ -FROM ubuntu:24.04 AS builder +FROM debian:bullseye AS builder ENV DEBIAN_FRONTEND=noninteractive +ENV CGIMAP_GITSHA=8ea707e10aeab5698e6859856111816d75354592 -RUN apt-get update -qq && \ - apt-get install -y gcc g++ make cmake \ - libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \ - libboost-program-options-dev libcrypto++-dev libyajl-dev \ - libpqxx-dev zlib1g-dev libfmt-dev git \ - ca-certificates \ - postgresql-16 postgresql-server-dev-all \ - wget \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install Argon2 from source -RUN wget https://github.com/P-H-C/phc-winner-argon2/archive/refs/tags/20190702.tar.gz && \ - tar -xzf 20190702.tar.gz && \ - cd phc-winner-argon2-20190702 && \ - make && \ - make install && \ - ldconfig - -WORKDIR /app - -# Copy the main application. -ENV CGIMAP_GITSHA=26cd7fa10affe5dbd13dbe16de34421059f53f18 -RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app && \ - cd /app && \ - git checkout $CGIMAP_GITSHA - -# Compile, install and remove source -RUN mkdir build && cd build && \ - CXXFLAGS="-Wall -Wextra -Wpedantic -Wno-unused-parameter" cmake .. -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release && \ - make -j$(nproc) && \ - ctest --output-on-failure && \ - strip openstreetmap-cgimap && \ - cp openstreetmap-cgimap /usr/local/bin - -RUN rm -rf /app/.git - -COPY start.sh / - -EXPOSE 8000 - -CMD ["/start.sh"] \ No newline at end of file +RUN apt-get update && apt-get install -y \ + build-essential cmake git \ + libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \ + libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \ + libargon2-dev libyajl-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git /cgimap \ + && cd /cgimap && git checkout $CGIMAP_GITSHA \ + && mkdir /cgimap/build && cd /cgimap/build \ + && cmake .. && cmake --build . + +FROM debian:bullseye-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + libxml2 libpqxx-6.4 libfcgi zlib1g libbrotli1 \ + libboost-program-options1.74.0 libfmt7 libmemcached11 libcrypto++8 \ + libargon2-1 libyajl2 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /cgimap/build/openstreetmap-cgimap /usr/local/bin/openstreetmap-cgimap + +ENTRYPOINT ["/usr/local/bin/openstreetmap-cgimap"] diff --git a/images/cgimap/lighttpd.conf.template b/images/cgimap/lighttpd.conf.template deleted file mode 100644 index b2724f73..00000000 --- a/images/cgimap/lighttpd.conf.template +++ /dev/null @@ -1,70 +0,0 @@ -# default document-root -server.document-root = "/openstreetmap/cgimap/" - -# TCP port -server.port = 80 - -server.reject-expect-100-with-417 = "disable" - -# selecting modules -server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias" ) - -# handling unknown routes -server.error-handler-404 = "/dispatch.map" - -# include, relative to dirname of main config file -#include "mime.types.conf" - -# read configuration from output of a command -#include_shell "/usr/local/bin/confmimetype /etc/mime.types" -mimetype.assign = ( - ".html" => "text/html", - ".txt" => "text/plain", - ".jpg" => "image/jpeg", - ".png" => "image/png" -) - -#debug.log-request-handling = "enable" - -$HTTP["request-method"] == "GET" { - url.rewrite-once = ( - "^/api/0\.6/map(\.(json|xml))?(\?(.*))?$" => "/dispatch.map", - "^/api/0\.6/(node|way|relation)/[[:digit:]]+(\.(json|xml))?$" => "/dispatch.map", - "^/api/0\.6/(node|way|relation)/[[:digit:]]+/history.*$" => "/dispatch.map", - "^/api/0\.6/(node|way|relation)/[[:digit:]]+/[[:digit:]]+.*$" => "/dispatch.map", - "^/api/0\.6/(node|way|relation)/[[:digit:]]+/relations$" => "/dispatch.map", - "^/api/0\.6/node/[[:digit:]]+/ways$" => "/dispatch.map", - "^/api/0\.6/(way|relation)/[[:digit:]]+/full$" => "/dispatch.map", - "^/api/0\.6/changeset/[[:digit:]]+.*$" => "/dispatch.map", - "^/api/0\.6/(nodes|ways|relations)(\?(.*))?$" => "/dispatch.map", - "^/api/0\.6/changeset/[[:digit:]]+/download$" => "/dispatch.map", - ) -} - -$HTTP["request-method"] == "POST" { - url.rewrite-once = ( - "^/api/0\.6/changeset/[[:digit:]]+/upload.*$" => "/dispatch.map", - ) -} - -$HTTP["request-method"] == "PUT" { - url.rewrite-once = ( - "^/api/0\.6/changeset/[[:digit:]]+/close.*$" => "/dispatch.map", - "^/api/0\.6/changeset/[[:digit:]]+$" => "/dispatch.map", - "^/api/0\.6/changeset/create.*$" => "/dispatch.map", - ) -} - -$HTTP["url"] =~ "^/(?!(dispatch\.map))" { - proxy.server = ( "" => ( ( "host" => "${API_WEB_HOST}", "port" => "${API_WEB_PORT}" ) ) ) -} - -fastcgi.debug = 1 - -fastcgi.server = ( ".map" => - (( "host" => "127.0.0.1", - "port" => 8000, - "check-local" => "disable", - )) -) - diff --git a/images/cgimap/start.sh b/images/cgimap/start.sh index 7db92bbe..726da9d7 100755 --- a/images/cgimap/start.sh +++ b/images/cgimap/start.sh @@ -1,30 +1,39 @@ #!/usr/bin/env bash + +export PGPASSWORD=$POSTGRES_PASSWORD export CGIMAP_HOST=$POSTGRES_HOST export CGIMAP_DBNAME=$POSTGRES_DB export CGIMAP_USERNAME=$POSTGRES_USER export CGIMAP_PASSWORD=$POSTGRES_PASSWORD export CGIMAP_OAUTH_HOST=$POSTGRES_HOST export CGIMAP_UPDATE_HOST=$POSTGRES_HOST +# Export CGIMAP configuration export CGIMAP_LOGFILE="/var/www/log/cgimap.log" -export CGIMAP_MEMCACHE=$OPENSTREETMAP_memcache_servers +export CGIMAP_MEMCACHE=$OPENSTREETMAP_MEMCACHE_SERVERS +# Average number of bytes/s to allow each client export CGIMAP_RATELIMIT="204800" -export CGIMAP_MAXDEBT="250" +# Maximum debt in MB to allow each client before rate limiting +export CGIMAP_MAXDEBT="2048" export CGIMAP_MAP_AREA="0.25" export CGIMAP_MAP_NODES="100000" export CGIMAP_MAX_WAY_NODES="2000" export CGIMAP_MAX_RELATION_MEMBERS="32000" # export CGIMAP_RATELIMIT_UPLOAD="true" -# export CGIMAP_MODERATOR_RATELIMIT="1048576" -# export CGIMAP_MODERATOR_MAXDEBT="1280" -# export CGIMAP_PIDFILE="/var/www/cgimap.pid" +export CGIMAP_MODERATOR_RATELIMIT="1048576" +export CGIMAP_MODERATOR_MAXDEBT="2048" -# Check for web site status if [[ "$WEBSITE_STATUS" == "database_readonly" || "$WEBSITE_STATUS" == "api_readonly" ]]; then export CGIMAP_DISABLE_API_WRITE="true" fi + if [[ "$WEBSITE_STATUS" == "database_offline" || "$WEBSITE_STATUS" == "api_offline" ]]; then - echo "Website is $WEBSITE_STATUS. No action required for cgimap service." + echo "Website is $WEBSITE_STATUS. No action required for cgimap service." else - /usr/local/bin/openstreetmap-cgimap --port=8000 --daemon --instances=10 && \ - tail --pid=$(cat /tmp/cgimap.pid) -f /dev/null + # PostgreSQL options to disable certain joins + export PGOPTIONS="-c enable_mergejoin=false -c enable_hashjoin=false" + # Display current PostgreSQL settings + psql -h $POSTGRES_HOST -U $POSTGRES_USER -c "SHOW enable_mergejoin;" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -c "SHOW enable_hashjoin;" + # Start the cgimap service + /usr/local/bin/openstreetmap-cgimap --port=8000 --daemon --instances=10 fi From fe68c7ef180f67d0bfd983498eaf0aa6810b6bcf Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 10 Apr 2025 12:50:36 -0500 Subject: [PATCH 10/25] Build cgimap image using multistage --- images/cgimap/Dockerfile | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index 1d438d4a..f4a7c50d 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -1,30 +1,27 @@ -FROM debian:bullseye AS builder - +FROM ruby:3.3.0 AS builder ENV DEBIAN_FRONTEND=noninteractive +ENV workdir=/var/www ENV CGIMAP_GITSHA=8ea707e10aeab5698e6859856111816d75354592 - RUN apt-get update && apt-get install -y \ - build-essential cmake git \ + build-essential cmake git-core curl file \ libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \ libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \ - libargon2-dev libyajl-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git /cgimap \ - && cd /cgimap && git checkout $CGIMAP_GITSHA \ - && mkdir /cgimap/build && cd /cgimap/build \ - && cmake .. && cmake --build . - -FROM debian:bullseye-slim - + libargon2-dev libyajl-dev libapache2-mod-fcgid \ + && rm -rf /var/lib/apt/lists/* +ENV cgimap=/tmp/openstreetmap-cgimap +RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ + && cd $cgimap \ + && git checkout $CGIMAP_GITSHA \ + && mkdir build && cd build && cmake .. && cmake --build . + +FROM ruby:3.3.0 ENV DEBIAN_FRONTEND=noninteractive - RUN apt-get update && apt-get install -y \ libxml2 libpqxx-6.4 libfcgi zlib1g libbrotli1 \ - libboost-program-options1.74.0 libfmt7 libmemcached11 libcrypto++8 \ - libargon2-1 libyajl2 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /cgimap/build/openstreetmap-cgimap /usr/local/bin/openstreetmap-cgimap + libboost-program-options1.74.0 libfmt-dev libmemcached11 libcrypto++8 \ + libargon2-1 libyajl2 libapache2-mod-fcgid \ + && rm -rf /var/lib/apt/lists/* -ENTRYPOINT ["/usr/local/bin/openstreetmap-cgimap"] +COPY --from=builder /tmp/openstreetmap-cgimap/build/openstreetmap-cgimap /usr/local/bin/openstreetmap-cgimap +RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local_libs.conf && ldconfig +CMD ["openstreetmap-cgimap"] From 96302978fd52b2b8094bba6ccf0ca7cdad292ee2 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 11 Apr 2025 12:27:06 -0500 Subject: [PATCH 11/25] Update cgimap config Update relese tag Pass env vars in web container for cgimap Update release tag Update tag name Update release tags --- .github/workflows/chartpress.yaml | 2 +- chartpress.yaml | 3 +- .../templates/cgimap/cgimap-deployment.yaml | 28 ++++--- osm-seed/templates/cgimap/cgimap-ingress.yaml | 26 ------ osm-seed/templates/cgimap/cgimap-service.yaml | 12 +-- osm-seed/templates/web/web-deployment.yaml | 12 +++ osm-seed/values.yaml | 79 ++++++++++--------- 7 files changed, 72 insertions(+), 90 deletions(-) delete mode 100644 osm-seed/templates/cgimap/cgimap-ingress.yaml diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index b33a4dd7..02eab1ae 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -42,6 +42,6 @@ jobs: then chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA else - chartpress --push --publish-chart + chartpress --push --publish-chart --tag fi \ No newline at end of file diff --git a/chartpress.yaml b/chartpress.yaml index 358d6a26..c50a3db4 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -30,8 +30,6 @@ charts: valuesPath: tilerServer.image tasking-manager-api: valuesPath: tmApi.image - # tiler-visor: - # valuesPath: tilerVisor.image nominatim: valuesPath: nominatimApi.image overpass-api: @@ -50,3 +48,4 @@ charts: valuesPath: planetFiles.image cgimap: valuesPath: cgimap.image + diff --git a/osm-seed/templates/cgimap/cgimap-deployment.yaml b/osm-seed/templates/cgimap/cgimap-deployment.yaml index 91dd7b31..67e37cd6 100644 --- a/osm-seed/templates/cgimap/cgimap-deployment.yaml +++ b/osm-seed/templates/cgimap/cgimap-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ template "osm-seed.fullname" . }}-cgimap + name: {{ .Release.Name }}-cgimap-deployment labels: app: {{ template "osm-seed.name" . }} component: cgimap-deployment @@ -14,27 +14,31 @@ spec: matchLabels: app: {{ template "osm-seed.name" . }} release: {{ .Release.Name }} - run: {{ .Release.Name }}-cgimap + run: {{ .Release.Name }}-cgimap-deployment template: metadata: labels: app: {{ template "osm-seed.name" . }} release: {{ .Release.Name }} - run: {{ .Release.Name }}-cgimap + run: {{ .Release.Name }}-cgimap-deployment spec: containers: - name: {{ .Chart.Name }}-cgimap image: "{{ .Values.cgimap.image.name }}:{{ .Values.cgimap.image.tag }}" ports: - name: http - containerPort: 80 + containerPort: 8000 protocol: TCP livenessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 600 # 10 min, because the compile process takes time. - timeoutSeconds: 30 + exec: + command: + - /bin/bash + - -c + - /liveness.sh + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 {{- if .Values.cgimap.resources.enabled }} resources: requests: @@ -49,14 +53,12 @@ spec: value: {{ .Release.Name }}-db - name: POSTGRES_DB value: {{ .Values.db.env.POSTGRES_DB }} + - name: PGPASSWORD + value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} - name: POSTGRES_PASSWORD value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} - name: POSTGRES_USER value: {{ .Values.db.env.POSTGRES_USER }} - - name: API_WEB_HOST - value: {{ .Release.Name }}-web - - name: API_WEB_PORT - value: 80 {{- if .Values.cgimap.nodeSelector.enabled }} nodeSelector: {{ .Values.cgimap.nodeSelector.label_key }} : {{ .Values.cgimap.nodeSelector.label_value }} diff --git a/osm-seed/templates/cgimap/cgimap-ingress.yaml b/osm-seed/templates/cgimap/cgimap-ingress.yaml deleted file mode 100644 index 7c158b60..00000000 --- a/osm-seed/templates/cgimap/cgimap-ingress.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if and .Values.cgimap.enabled (eq .Values.serviceType "ClusterIP") }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ template "osm-seed.fullname" . }}-ingress-cgimap-api - annotations: - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod-issuer -spec: - tls: - - hosts: - - cgimap.{{ .Values.domain }} - secretName: {{ template "osm-seed.fullname" . }}-secret-cgimap - - rules: - - host: cgimap.{{ .Values.domain }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ template "osm-seed.fullname" . }}-cgimap - port: - number: 80 -{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/cgimap/cgimap-service.yaml b/osm-seed/templates/cgimap/cgimap-service.yaml index a7449407..bed7721a 100644 --- a/osm-seed/templates/cgimap/cgimap-service.yaml +++ b/osm-seed/templates/cgimap/cgimap-service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "osm-seed.fullname" . }}-cgimap + name: {{ .Release.Name }}-cgimap-service labels: app: {{ template "osm-seed.name" . }} component: cgimap-service @@ -30,14 +30,8 @@ spec: targetPort: http protocol: TCP name: http - {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} - - port: 443 - targetPort: http - protocol: TCP - name: https - {{- end }} selector: app: {{ template "osm-seed.name" . }} release: {{ .Release.Name }} - run: {{ .Release.Name }}-cgimap -{{- end }} \ No newline at end of file + run: {{ .Release.Name }}-cgimap-deployment +{{- end }} diff --git a/osm-seed/templates/web/web-deployment.yaml b/osm-seed/templates/web/web-deployment.yaml index 43bf25cf..ffcb965f 100644 --- a/osm-seed/templates/web/web-deployment.yaml +++ b/osm-seed/templates/web/web-deployment.yaml @@ -154,6 +154,18 @@ spec: value: {{ .Values.web.env.RAILS_STORAGE_REGION | quote }} - name: RAILS_STORAGE_BUCKET value: {{ .Values.web.env.RAILS_STORAGE_BUCKET | quote }} + # Cgimap run in external container + {{- if and .Values.web.env.EXTERNAL .Values.cgimap.enabled }} + - name: CGIMAP_URL + value: {{ .Release.Name }}-cgimap-service + - name: CGIMAP_PORT + value: {{ quote 80 }} + {{- else }} + - name: CGIMAP_URL + value: "127.0.0.1" + - name: CGIMAP_PORT + value: {{ quote 8000 }} + {{- end }} volumeMounts: - mountPath: /dev/shm name: shared-memory diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 5c129a20..c6cd09c3 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -127,7 +127,7 @@ db: max_worker_processes = 4 max_parallel_workers_per_gather = 2 max_parallel_workers = 4 - + # ==================================================================================================== # Variables for osm-seed website # ==================================================================================================== @@ -161,6 +161,7 @@ web: RAILS_STORAGE_SERVICE: local RAILS_STORAGE_REGION: us-east-1 RAILS_STORAGE_BUCKET: osmseed-website-bucket + EXTERNAL_CGIMAP: false resources: enabled: false requests: @@ -200,6 +201,23 @@ memcached: enabled: false memory: "2Gi" cpu: "2" + +# ==================================================================================================== +# Cgimap +# ==================================================================================================== +cgimap: + enabled: false + priorityClass: "low-priority" + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false # ==================================================================================================== # Variables for full-history container # ==================================================================================================== @@ -460,7 +478,8 @@ tilerImposm: env: TILER_IMPORT_FROM: osm TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf - TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson + TILER_IMPORT_LIMIT: + https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson TILER_CACHE_AWS_ENDPOINT: "" UPLOAD_EXPIRED_FILES: true IMPORT_NATURAL_EARTH: true @@ -718,7 +737,8 @@ nominatimApi: THREADS: 4 NOMINATIM_PASSWORD: psw1234 PGDATA: /var/lib/postgresql/14/main - NOMINATIM_ADDRESS_LEVEL_CONFIG_URL: https://gist.githubusercontent.com/lonvia/8502c61b3dd159e67eb8be8368d864c1/raw/d05a1d681230632bb2e35cbf363f2baec8acd657/address-levels.json + NOMINATIM_ADDRESS_LEVEL_CONFIG_URL: + https://gist.githubusercontent.com/lonvia/8502c61b3dd159e67eb8be8368d864c1/raw/d05a1d681230632bb2e35cbf363f2baec8acd657/address-levels.json UPDATE_MODE: continuous #Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) OSMSEED_WEB_API_DOMAIN: www.openstreetmap.org resources: @@ -764,7 +784,8 @@ overpassApi: OVERPASS_PLANET_URL: http://download.geofabrik.de/europe/monaco-latest.osm.bz2 OVERPASS_DIFF_URL: http://download.openstreetmap.fr/replication/europe/monaco/minute/ OVERPASS_RULES_LOAD: 10 - OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' + OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium + cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' OVERPASS_REPLICATION_SEQUENCE_NUMBER: 5201000 OVERPASS_ALLOW_DUPLICATE_QUERIES: "yes" persistenceDisk: @@ -834,20 +855,20 @@ taginfo: label_key: nodegroup_type label_value: web cronjob: + enabled: true + schedule: "0 2 */3 * *" + nodeSelector: enabled: true - schedule: "0 2 */3 * *" - nodeSelector: - enabled: true - label_key: nodegroup_type - label_value: job_xlarge - resources: - enabled: false - requests: - memory: "13Gi" - cpu: "3600m" - limits: - memory: "14Gi" - cpu: "3800m" + label_key: nodegroup_type + label_value: job_xlarge + resources: + enabled: false + requests: + memory: "13Gi" + cpu: "3600m" + limits: + memory: "14Gi" + cpu: "3800m" # ==================================================================================================== # Variables for osm-simple-metrics # ==================================================================================================== @@ -1044,25 +1065,5 @@ adiffService: planetFiles: enabled: false image: - name: "" - tag: "" -# ==================================================================================================== -# Cgimap -# ==================================================================================================== -cgimap: - enabled: false - priorityClass: "low-priority" - image: - name: '' - tag: '' - resources: - enabled: false - requests: - memory: '20Gi' - cpu: '8' - limits: - memory: '24Gi' - cpu: '10' - nodeSelector: - enabled: false - + name: "developmentseed/osmseed-planet-files" + tag: "0.1.0-0.dev.git.966.hc380960" From 89acb3b9642398ea86b45c774336945ae944cdc0 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 11 Apr 2025 16:36:01 -0500 Subject: [PATCH 12/25] Update release tags --- .github/workflows/chartpress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 02eab1ae..b33a4dd7 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -42,6 +42,6 @@ jobs: then chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA else - chartpress --push --publish-chart --tag + chartpress --push --publish-chart fi \ No newline at end of file From 7d9409def21f8d35f4e86ab72264c1a76b96124f Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 10:39:30 -0500 Subject: [PATCH 13/25] Add memcache server for cgimap --- osm-seed/templates/cgimap/cgimap-deployment.yaml | 4 ++++ osm-seed/templates/web/web-deployment.yaml | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/osm-seed/templates/cgimap/cgimap-deployment.yaml b/osm-seed/templates/cgimap/cgimap-deployment.yaml index 67e37cd6..02cd77b8 100644 --- a/osm-seed/templates/cgimap/cgimap-deployment.yaml +++ b/osm-seed/templates/cgimap/cgimap-deployment.yaml @@ -59,6 +59,10 @@ spec: value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} - name: POSTGRES_USER value: {{ .Values.db.env.POSTGRES_USER }} + - name: WEBSITE_STATUS + value: {{ .Values.web.env.WEBSITE_STATUS }} + - name: OPENSTREETMAP_MEMCACHE_SERVERS + value: {{ .Release.Name }}-memcached:11211 {{- if .Values.cgimap.nodeSelector.enabled }} nodeSelector: {{ .Values.cgimap.nodeSelector.label_key }} : {{ .Values.cgimap.nodeSelector.label_value }} diff --git a/osm-seed/templates/web/web-deployment.yaml b/osm-seed/templates/web/web-deployment.yaml index ffcb965f..e707c28c 100644 --- a/osm-seed/templates/web/web-deployment.yaml +++ b/osm-seed/templates/web/web-deployment.yaml @@ -155,7 +155,9 @@ spec: - name: RAILS_STORAGE_BUCKET value: {{ .Values.web.env.RAILS_STORAGE_BUCKET | quote }} # Cgimap run in external container - {{- if and .Values.web.env.EXTERNAL .Values.cgimap.enabled }} + {{- if and .Values.web.env.EXTERNAL_CGIMAP .Values.cgimap.enabled }} + - name: EXTERNAL_CGIMAP + value: {{ .Values.web.env.EXTERNAL_CGIMAP | quote }} - name: CGIMAP_URL value: {{ .Release.Name }}-cgimap-service - name: CGIMAP_PORT From a7c94185f323217b6bdbc0813783b51005a9f8c8 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 10:51:01 -0500 Subject: [PATCH 14/25] Update cgimap start and github action --- .github/workflows/chartpress.yaml | 80 +++++++++++++++---------------- images/cgimap/Dockerfile | 44 +++++++++++------ 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index b33a4dd7..40c79565 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -4,44 +4,42 @@ jobs: build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Setup Git - run: | - git config --global user.email "noreply@developmentseed.org" - git config --global user.name "Github Action" - - name: Install Chartpress and Python modules - run: | - pip install -v chartpress six yq - - name: Set env vars for publish charts - run: | - GIT_BRANCH_NAME=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - GIT_BRANCH_NAME=${GIT_BRANCH_NAME/\//_} - OSM_SEED_VERSION=$(yq -r .appVersion ./osm-seed/Chart.yaml) - SHORT_GITHUB_SHA=${GITHUB_SHA::7} - echo "GIT_BRANCH_NAME=$GIT_BRANCH_NAME" >> $GITHUB_ENV - echo "OSM_SEED_VERSION=$OSM_SEED_VERSION" >> $GITHUB_ENV - echo "SHORT_GITHUB_SHA=$SHORT_GITHUB_SHA" >> $GITHUB_ENV - - name: Run Chartpress - env: - GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }} - GITHUB_EVENT_NAME: ${{ github.event_name }} - run: | - if [[ $GIT_BRANCH_NAME = "main" ]] - then - chartpress --push --publish-chart --tag ${OSM_SEED_VERSION} - elif [[ $GIT_BRANCH_NAME = "develop" ]] - then - chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA - else - chartpress --push --publish-chart - fi - \ No newline at end of file + - uses: actions/checkout@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Setup git + run: git config --global user.email "noreply@developmentseed.org" && git config --global user.name "Github Action" + - name: Install Chartpress and some python modules + run: | + pip install -v chartpress && pip install six && pip install yq + - name: Set env vars for publish charts + run: | + GIT_BRANCH_NAME=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + GIT_BRANCH_NAME=${GIT_BRANCH_NAME/\//_} + OSM_SEED_VERSION=$(yq -r .appVersion ./osm-seed/Chart.yaml) + SHORT_GITHUB_SHA=${GITHUB_SHA::7} + echo "GIT_BRANCH_NAME=$GIT_BRANCH_NAME" >> $GITHUB_ENV + echo "OSM_SEED_VERSION=$OSM_SEED_VERSION" >> $GITHUB_ENV + echo "SHORT_GITHUB_SHA=$SHORT_GITHUB_SHA" >> $GITHUB_ENV + - name: Run Chartpress + env: + GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + run: | + if [[ $GIT_BRANCH_NAME = "main" ]] + then + chartpress --push --publish-chart --tag ${OSM_SEED_VERSION} + elif [[ $GIT_BRANCH_NAME = "develop" ]] + then + chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA + else + chartpress --push --publish-chart + fi + \ No newline at end of file diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index f4a7c50d..660cfec0 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -1,27 +1,41 @@ FROM ruby:3.3.0 AS builder + ENV DEBIAN_FRONTEND=noninteractive ENV workdir=/var/www ENV CGIMAP_GITSHA=8ea707e10aeab5698e6859856111816d75354592 -RUN apt-get update && apt-get install -y \ - build-essential cmake git-core curl file \ +ENV cgimap=/tmp/openstreetmap-cgimap + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git curl file \ libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \ libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \ - libargon2-dev libyajl-dev libapache2-mod-fcgid \ - && rm -rf /var/lib/apt/lists/* -ENV cgimap=/tmp/openstreetmap-cgimap -RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ - && cd $cgimap \ - && git checkout $CGIMAP_GITSHA \ - && mkdir build && cd build && cmake .. && cmake --build . + libargon2-dev libyajl-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ + && cd $cgimap \ + && git checkout $CGIMAP_GITSHA \ + && mkdir build && cd build && cmake .. && cmake --build . + + +FROM ruby:3.3.0-slim -FROM ruby:3.3.0 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ + +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl gnupg && \ + echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ libxml2 libpqxx-6.4 libfcgi zlib1g libbrotli1 \ libboost-program-options1.74.0 libfmt-dev libmemcached11 libcrypto++8 \ - libargon2-1 libyajl2 libapache2-mod-fcgid \ - && rm -rf /var/lib/apt/lists/* + libargon2-1 libyajl2 libapache2-mod-fcgid postgresql-client \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* COPY --from=builder /tmp/openstreetmap-cgimap/build/openstreetmap-cgimap /usr/local/bin/openstreetmap-cgimap -RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local_libs.conf && ldconfig -CMD ["openstreetmap-cgimap"] + +COPY *.sh / +RUN chmod +x /*.sh && echo "/usr/local/lib" > /etc/ld.so.conf.d/local_libs.conf && ldconfig + +CMD ["/start.sh"] From 1095b8595b7a7e9c0ea5e6aa0a19b2fc5c5a89f4 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 11:11:23 -0500 Subject: [PATCH 15/25] Add --daemon for cgimap --- images/cgimap/liveness.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 images/cgimap/liveness.sh diff --git a/images/cgimap/liveness.sh b/images/cgimap/liveness.sh new file mode 100755 index 00000000..bce94595 --- /dev/null +++ b/images/cgimap/liveness.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +pgrep -f openstreetmap-cgimap > /dev/null +cgimap_status=$? + +# Check PostgreSQL connection +PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 1;" > /dev/null 2>&1 +postgres_status=$? + +# Exit code logic +if [ $cgimap_status -eq 0 ] && [ $postgres_status -eq 0 ]; then + echo "cgimap and PostgreSQL are healthy" + exit 0 +else + [ $cgimap_status -ne 0 ] && echo "cgimap not running" >&2 + [ $postgres_status -ne 0 ] && echo "cannot connect to PostgreSQL" >&2 + exit 1 +fi \ No newline at end of file From 8dac9a737aacbc7a8eb736fff77d23cab494a97c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 11:44:28 -0500 Subject: [PATCH 16/25] Set empty values for planetFiles.image --- osm-seed/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index c6cd09c3..cc51d77a 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -1065,5 +1065,5 @@ adiffService: planetFiles: enabled: false image: - name: "developmentseed/osmseed-planet-files" - tag: "0.1.0-0.dev.git.966.hc380960" + name: "" + tag: "" From 2e179bde4c52c40852b2f2e9ee40ba5420e18774 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 11:55:03 -0500 Subject: [PATCH 17/25] Disable cgimap in chartpress --- chartpress.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chartpress.yaml b/chartpress.yaml index c50a3db4..0665398f 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -46,6 +46,5 @@ charts: valuesPath: osmchaDb.image planet-files: valuesPath: planetFiles.image - cgimap: - valuesPath: cgimap.image - + # cgimap: + # valuesPath: cgimap.image From 42798edc9ba4ede35968e95fae155a6bffcb3d05 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 14 Apr 2025 19:28:19 -0500 Subject: [PATCH 18/25] Use chartpress==2.3.0 --- .github/workflows/chartpress.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 40c79565..77c94e43 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -13,12 +13,12 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.7' - name: Setup git run: git config --global user.email "noreply@developmentseed.org" && git config --global user.name "Github Action" - name: Install Chartpress and some python modules run: | - pip install -v chartpress && pip install six && pip install yq + pip install -v chartpress==2.3.0 && pip install six && pip install yq - name: Set env vars for publish charts run: | GIT_BRANCH_NAME=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} From 6549d9cc806c961abbe1e99502290678dc81ee92 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 15 Apr 2025 14:27:15 -0500 Subject: [PATCH 19/25] Add HorizontalPodAutoscaler for cgimap --- osm-seed/templates/cgimap/cgimap-hpa.yaml | 26 +++++++++++++++++++++++ osm-seed/values.yaml | 9 ++++++++ 2 files changed, 35 insertions(+) create mode 100644 osm-seed/templates/cgimap/cgimap-hpa.yaml diff --git a/osm-seed/templates/cgimap/cgimap-hpa.yaml b/osm-seed/templates/cgimap/cgimap-hpa.yaml new file mode 100644 index 00000000..4fd5a128 --- /dev/null +++ b/osm-seed/templates/cgimap/cgimap-hpa.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.cgimap.enabled .Values.cgimap.autoscaling.enabled -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Release.Name }}-cgimmap-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Release.Name }}-cgimap-deployment + minReplicas: {{ .Values.cgimap.autoscaling.minReplicas }} + maxReplicas: {{ .Values.cgimap.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.cgimap.autoscaling.cpuUtilization }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.cgimap.autoscaling.memoryUtilization }} +{{- end }} \ No newline at end of file diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index cc51d77a..14a3d8a0 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -206,6 +206,9 @@ memcached: # Cgimap # ==================================================================================================== cgimap: + image: + name: "" + tag: "" enabled: false priorityClass: "low-priority" resources: @@ -218,6 +221,12 @@ cgimap: cpu: '10' nodeSelector: enabled: false + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 4 + cpuUtilization: 60 + memoryUtilization: 60 # ==================================================================================================== # Variables for full-history container # ==================================================================================================== From deadff6ef7173f281a8b49f0de57fc38f3db7259 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 16 Apr 2025 12:24:13 -0500 Subject: [PATCH 20/25] Update web-api container --- images/cgimap/start.sh | 2 +- images/web/Dockerfile | 187 ++++++++-------- images/web/config/production.conf | 50 +++-- images/web/config/settings.yml | 43 ++-- images/web/liveness.sh | 19 +- images/web/start.sh | 208 +++++++++++------- .../templates/cgimap/cgimap-deployment.yaml | 3 +- osm-seed/templates/cgimap/cgimap-hpa.yaml | 2 +- 8 files changed, 292 insertions(+), 222 deletions(-) diff --git a/images/cgimap/start.sh b/images/cgimap/start.sh index 726da9d7..78425dbc 100755 --- a/images/cgimap/start.sh +++ b/images/cgimap/start.sh @@ -9,7 +9,7 @@ export CGIMAP_OAUTH_HOST=$POSTGRES_HOST export CGIMAP_UPDATE_HOST=$POSTGRES_HOST # Export CGIMAP configuration export CGIMAP_LOGFILE="/var/www/log/cgimap.log" -export CGIMAP_MEMCACHE=$OPENSTREETMAP_MEMCACHE_SERVERS +export CGIMAP_MEMCACHE=$MEMCACHE_SERVER # Average number of bytes/s to allow each client export CGIMAP_RATELIMIT="204800" # Maximum debt in MB to allow each client before rate limiting diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 927e49da..0927843b 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -1,111 +1,98 @@ -FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND=noninteractive -ENV workdir /var/www - -# Production OSM setup -ENV RAILS_ENV=production - -# Install the openstreetmap-website dependencies -RUN apt-get update \ - && apt-get install -y \ - ruby ruby-dev ruby-bundler libmagickwand-dev libxml2-dev libxslt1-dev \ - apache2 apache2-dev build-essential git-core postgresql-client \ - libpq-dev libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev curl \ - default-jre-headless file gpg-agent libvips-dev locales software-properties-common tzdata unzip \ - advancecomp gifsicle libjpeg-progs jhead jpegoptim optipng pngcrush pngquant libyaml-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -## Install node -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update && apt-get install -y nodejs yarn && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install openstreetmap-cgimap requirements -RUN apt-get update && apt-get -y install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \ - libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \ - libargon2-dev libyajl-dev cmake libapache2-mod-fcgid && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install cgimap, before remove basic auth -ENV cgimap /openstreetmap-cgimap -ENV CGIMAP_GITSHA=26cd7fa10affe5dbd13dbe16de34421059f53f18 -RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ - && cd $cgimap \ - && git checkout $CGIMAP_GITSHA \ - && rm -rf .git \ - && mkdir build \ - && cd build \ - && cmake .. \ - && cmake --build . - -# Install svgo required -RUN npm install -g svgo - -# Install openstreetmap-website -RUN rm -rf $workdir/html - -## Sep 2023 -ENV OPENSTREETMAP_WEBSITE_GITSHA=d23763d6cdbf5ec11f0e83f8e6e8fb32ed973e6a -RUN curl -L https://github.com/openstreetmap/openstreetmap-website/archive/$OPENSTREETMAP_WEBSITE_GITSHA.zip --output website.zip && unzip website.zip -RUN mv openstreetmap-website-$OPENSTREETMAP_WEBSITE_GITSHA/* $workdir/ -WORKDIR $workdir - -# Install Ruby packages -RUN gem install bundler && bundle install - -# Configure database.yml and secrets.yml -RUN cp $workdir/config/example.database.yml $workdir/config/database.yml -RUN touch $workdir/config/settings.local.yml -RUN cp $workdir/config/example.storage.yml $workdir/config/storage.yml -RUN echo "#session key \n\ -production: \n\ - secret_key_base: $(rails secret)" > $workdir/config/secrets.yml -# Protect sensitive information -RUN chmod 600 $workdir/config/database.yml $workdir/config/secrets.yml -RUN bundle exec bin/yarn install +FROM ruby:3.3-slim AS builder -RUN rails i18n:js:export assets:precompile - -# The rack interface requires a `tmp` directory to use openstreetmap-cgimap -RUN ln -s /tmp /var/www/tmp +ENV DEBIAN_FRONTEND=noninteractive \ + workdir=/var/www -# Add Apache configuration file -ADD config/production.conf /etc/apache2/sites-available/production.conf -RUN a2enmod headers -RUN a2enmod setenvif -RUN a2dissite 000-default -RUN a2ensite production +WORKDIR $workdir -# Install Passenger + Apache module -RUN apt-key adv --refresh-keys --keyserver keyserver.ubuntu.com -RUN apt-get update && apt-get install -y libapache2-mod-passenger lighttpd +# Install base build dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git curl gnupg build-essential \ + libarchive-dev zlib1g-dev libcurl4-openssl-dev \ + apache2 apache2-dev libapache2-mod-passenger libapache2-mod-fcgid libapr1-dev libaprutil1-dev \ + postgresql-client libpq-dev libxml2-dev libyaml-dev \ + pngcrush optipng advancecomp pngquant jhead jpegoptim gifsicle libjpeg-progs \ + && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + && apt-get install -y nodejs \ + && npm install -g yarn svgo \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -# Enable the Passenger Apache module and restart Apache -RUN echo "ServerName $(cat /etc/hostname)" >> /etc/apache2/apache2.conf RUN a2enmod passenger -# Check installation -RUN /usr/bin/passenger-config validate-install -RUN /usr/sbin/passenger-memory-stats - -# Enable required apache modules for the cgimap Apache service -RUN a2enmod proxy proxy_http rewrite lbmethod_byrequests proxy_fcgi +# Clone OSM Website +ENV OPENSTREETMAP_WEBSITE_GITSHA=ea3760f94d9d74d3aaa8492182b9e1a15ec1effa +RUN rm -rf $workdir/* && \ + git clone https://github.com/openstreetmap/openstreetmap-website.git $workdir && \ + cd $workdir && \ + git checkout $OPENSTREETMAP_WEBSITE_GITSHA && \ + git fetch && rm -rf .git + +# Install Ruby/Node dependencies +RUN gem install bundler && \ + bundle install && \ + yarn install && \ + bundle exec rake yarn:install + +# Dummy config for precompile +RUN cp config/example.database.yml config/database.yml && \ + cp config/example.storage.yml config/storage.yml && \ + touch config/settings.local.yml && \ + chmod 600 config/database.yml + +# Create dummy credentials +RUN rm -f config/credentials.yml.enc && \ + export RAILS_MASTER_KEY=$(openssl rand -hex 16) && \ + export SECRET_KEY_BASE=$(bundle exec rails secret) && \ + echo $RAILS_MASTER_KEY > config/master.key && \ + EDITOR="echo" RAILS_MASTER_KEY=$RAILS_MASTER_KEY rails credentials:edit && \ + RAILS_MASTER_KEY=$RAILS_MASTER_KEY rails runner "\ + require 'active_support/encrypted_configuration'; \ + require 'yaml'; \ + creds = ActiveSupport::EncryptedConfiguration.new(\ + config_path: 'config/credentials.yml.enc', \ + key_path: 'config/master.key', \ + env_key: 'RAILS_MASTER_KEY', \ + raise_if_missing_key: true \ + ); \ + credentials = { secret_key_base: '$SECRET_KEY_BASE' }; \ + creds.write(credentials.to_yaml); \ + puts 'Credentials configured correctly.'" + +# Precompile assets +RUN bundle exec rake i18n:js:export && \ + bundle exec rake assets:precompile + +FROM ruby:3.3-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + workdir=/var/www -# Config the virtual host apache2 -RUN apache2ctl configtest +WORKDIR $workdir -# Set Permissions for www-data -RUN chown -R www-data: $workdir +# Install only runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + apache2 libapache2-mod-passenger libapache2-mod-fcgid \ + libpq5 libxml2 libyaml-0-2 libarchive13 file libgd-dev \ + postgresql-client curl \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -# Add settings -ADD config/settings.yml $workdir/config/ +COPY --from=builder /var/www /var/www +COPY --from=builder /usr/local/bundle /usr/local/bundle -COPY start.sh $workdir/ -COPY liveness.sh $workdir/ +# Symlink tmp for Passenger +RUN ln -s /tmp /var/www/tmp -CMD $workdir/start.sh +# Apache configuration +COPY config/production.conf /etc/apache2/sites-available/production.conf +RUN a2enmod headers setenvif proxy proxy_http proxy_fcgi fcgid rewrite lbmethod_byrequests passenger && \ + a2dissite 000-default && \ + a2ensite production && \ + echo "ServerName localhost" >> /etc/apache2/apache2.conf && \ + apache2ctl configtest + +COPY config/settings.yml $workdir/config/ +COPY start.sh liveness.sh $workdir/ +RUN chmod +x $workdir/*.sh +RUN chown -R www-data:www-data /var/www +CMD ["./start.sh"] diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 89f73d8f..718c4779 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -2,36 +2,39 @@ # ServerName localhost # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public - PassengerRuby /usr/bin/ruby + PassengerRuby /usr/local/bin/ruby RewriteEngine On + + # Redirect to HTTPS RewriteCond %{HTTP:X-Forwarded-Proto} =http - # Development mode in case domain is localhost - # ======Redirect to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTP_HOST} !=127.0.0.1 RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect to wwww osmseed.org - # RewriteCond %{HTTP_HOST} =osmseed.org - # RewriteCond %{HTTP_HOST} !^www\. [NC] - # RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + ; # Redirect to www openstreetmap.org + ; RewriteCond %{HTTP_HOST} =openstreetmap.org + ; RewriteCond %{HTTP_HOST} !^www\. [NC] + ; RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - CGIPassAuth On + CGIPassAuth On + SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 - # ======Proxying traffic to CGImap==== + # Proxying traffic to CGImap + ProxyTimeout 1200 RewriteCond %{REQUEST_URI} ^/api/0\.6/map - RewriteRule ^/api/0\.6/map(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] + RewriteRule ^/api/0\.6/map(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ - RewriteRule ^/api/0\.6/(node|way|relation|changeset)/[0-9]+(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/history(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] - RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation|changeset)/[0-9]+(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/history(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] + RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] # Relax Apache security settings @@ -39,4 +42,15 @@ Allow from all Options -MultiViews - + + # Additional FastCGI configurations + + ProxyTimeout 1200 + ProxyBadHeader Ignore + + + + FcgidIOTimeout 1200 + FcgidConnectTimeout 1200 + + diff --git a/images/web/config/settings.yml b/images/web/config/settings.yml index ce42d9e5..3e145e09 100644 --- a/images/web/config/settings.yml +++ b/images/web/config/settings.yml @@ -1,6 +1,6 @@ # The server protocol and host server_protocol: "http" -server_url: "openstreetmap.example.com" +server_url: "openstreetmap.example.com" # Publisher #publisher_url: "" # The generator @@ -32,7 +32,7 @@ default_changeset_query_limit: 100 # Maximum limit on the number of changesets returned by the changeset query api method max_changeset_query_limit: 100 # Maximum number of nodes that will be returned by the api in a map request -max_number_of_nodes: 50000 +max_number_of_nodes: 100000 # Maximum number of nodes that can be in a way (checked on save) max_number_of_way_nodes: 2000 # Maximum number of members that can be in a relation (checked on save) @@ -50,21 +50,26 @@ max_trace_size: 1000000 # Zoom level to use for postcode results from the geocoder postcode_zoom: 15 # Timeout for API calls in seconds -api_timeout: 300 +api_timeout: 600 # Timeout for web pages in seconds -web_timeout: 30 +web_timeout: 600 # Periods (in hours) which are allowed for user blocks user_block_periods: [0, 1, 3, 6, 12, 24, 48, 96, 168, 336, 731, 4383, 8766, 87660] # Account deletion cooldown period (in hours) since last changeset close; null to disable, 0 to make sure there aren't any open changesets when the deletion happens user_account_deletion_delay: null # Rate limit for message sending max_messages_per_hour: 60 +# Default limit on the number of messages returned by inbox and outbox message api +default_message_query_limit: 100 +# Maximum number of messages returned by inbox and outbox message api +max_message_query_limit: 100 # Rate limit for friending max_friends_per_hour: 60 # Rate limit for changeset comments min_changeset_comments_per_hour: 1 initial_changeset_comments_per_hour: 6 max_changeset_comments_per_hour: 60 +comments_to_max_changeset_comments: 200 moderator_changeset_comments_per_hour: 36000 # Rate limit for changes min_changes_per_hour: 100 @@ -73,6 +78,13 @@ max_changes_per_hour: 100000 days_to_max_changes: 7 importer_changes_per_hour: 1000000 moderator_changes_per_hour: 1000000 +# Size limit for changes +min_size_limit: 10000000 +initial_size_limit: 30000000 +max_size_limit: 5400000000 +days_to_max_size_limit: 28 +importer_size_limit: 5400000000 +moderator_size_limit: 5400000000 # Domain for handling message replies #messages_domain: "messages.openstreetmap.org" # MaxMind GeoIPv2 database @@ -94,25 +106,26 @@ attachments_dir: ":rails_root/public/attachments" # Log file to use for logstash #logstash_path: "" # List of memcache servers to use for caching -#memcache_servers: [] -# Enable HTTP basic authentication support -basic_auth_support: true -# Enable legacy OAuth 1.0 support -oauth_10_support: true -oauth_10_registration: true +memcache_servers: [] # URL of Nominatim instance to use for geocoding -nominatim_url: "https://nominatim.openstreetmap.org/" +nominatim_url: "https://nominatim-api.openstreetmap.org/" # Default editor default_editor: "id" # OAuth application for the web site -oauth_application: "OAUTH_CLIENT_ID" -oauth_key: "OAUTH_KEY" +oauth_application: "" +oauth_key: "" # OAuth application for iD id_application: "" # Imagery to return in capabilities as blacklisted -imagery_blacklist: [] +imagery_blacklist: + # Current Google imagery URLs have google or googleapis in the domain + - ".*\\.google(apis)?\\..*/.*" + # Blacklist VWorld + - "http://xdworld\\.vworld\\.kr:8080/.*" + # Blacklist here + - ".*\\.here\\.com[/:].*" # URL of Overpass instance to use for feature queries -overpass_url: "https://overpass-api.de/api/interpreter" +overpass_url: "https://overpass-api.openstreetmap.org/api/interpreter" overpass_credentials: false # Routing endpoints graphhopper_url: "https://graphhopper.com/api/1/route" diff --git a/images/web/liveness.sh b/images/web/liveness.sh index 331df5b9..27af7663 100755 --- a/images/web/liveness.sh +++ b/images/web/liveness.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# This is a script for evaluating if openstreetmap-cgimap, apache2, and PostgreSQL are running in the container. +# This is a script for evaluating if apache2 is running in the container and PostgreSQL is reachable. check_process() { if ps aux | grep "$1" | grep -v grep > /dev/null; then return 0 @@ -8,29 +8,24 @@ check_process() { fi } -# Check for openstreetmap-cgimap process -check_process "/openstreetmap-cgimap/build/openstreetmap-cgimap" -cgimap_status=$? - # Check for apache2 process check_process "apache2" apache_status=$? # Check PostgreSQL connection check_postgres() { - PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB -c "SELECT 1;" > /dev/null 2>&1 + PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 1;" > /dev/null 2>&1 return $? } check_postgres postgres_status=$? -if [ $cgimap_status -eq 0 ] && [ $apache_status -eq 0 ] && [ $postgres_status -eq 0 ]; then - echo "All services (openstreetmap-cgimap, apache2, PostgreSQL) are running." +if [ $apache_status -eq 0 ] && [ $postgres_status -eq 0 ]; then + echo "Apache and PostgreSQL are running." exit 0 else - [ $cgimap_status -ne 0 ] && echo "openstreetmap-cgimap is not running!" 1>&2 - [ $apache_status -ne 0 ] && echo "apache2 is not running!" 1>&2 - [ $postgres_status -ne 0 ] && echo "Failed to connect to PostgreSQL!" 1>&2 + [ $apache_status -ne 0 ] && echo "apache2 is not running!" >&2 + [ $postgres_status -ne 0 ] && echo "Failed to connect to PostgreSQL!" >&2 exit 1 -fi \ No newline at end of file +fi diff --git a/images/web/start.sh b/images/web/start.sh index c559b499..a379fddb 100755 --- a/images/web/start.sh +++ b/images/web/start.sh @@ -2,85 +2,145 @@ workdir="/var/www" export RAILS_ENV=production -#### SETTING UP THE PRODUCTION DATABASE -echo " # Production DB +setup_env_vars() { + #### Setting up the production database + cat < "$workdir/config/database.yml" production: adapter: postgresql host: ${POSTGRES_HOST} database: ${POSTGRES_DB} username: ${POSTGRES_USER} password: ${POSTGRES_PASSWORD} - encoding: utf8" >$workdir/config/database.yml - -#### SETTING UP SERVER_URL AND SERVER_PROTOCOL -sed -i -e 's/server_url: "openstreetmap.example.com"/server_url: "'$SERVER_URL'"/g' $workdir/config/settings.yml -sed -i -e 's/server_protocol: "http"/server_protocol: "'$SERVER_PROTOCOL'"/g' $workdir/config/settings.yml - -#### SETTING UP MAIL SENDER -sed -i -e 's/smtp_address: "localhost"/smtp_address: "'$MAILER_ADDRESS'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_domain: "localhost"/smtp_domain: "'$MAILER_DOMAIN'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_enable_starttls_auto: false/smtp_enable_starttls_auto: true/g' $workdir/config/settings.yml -sed -i -e 's/smtp_authentication: null/smtp_authentication: "login"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_user_name: null/smtp_user_name: "'$MAILER_USERNAME'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_password: null/smtp_password: "'$MAILER_PASSWORD'"/g' $workdir/config/settings.yml -sed -i -e 's/openstreetmap@example.com/'$MAILER_FROM'/g' $workdir/config/settings.yml -sed -i -e 's/smtp_port: 25/smtp_port: '$MAILER_PORT'/g' $workdir/config/settings.yml - -### SETTING UP UP OAUTH-2 ID KEY FOR iD -sed -i -e 's/id_application: ""/id_application: "'$OPENSTREETMAP_id_key'"/g' $workdir/config/settings.yml - -### SETTING UP OAUTH-2 ID KEY WEBSITE -sed -i -e 's/OAUTH_CLIENT_ID/'$OAUTH_CLIENT_ID'/g' $workdir/config/settings.yml -sed -i -e 's/OAUTH_KEY/'$OAUTH_KEY'/g' $workdir/config/settings.yml - -#### SETTING UP ENV VARS FOR MEMCACHED SERVER -sed -i -e 's/#memcache_servers: \[\]/memcache_servers: "'$OPENSTREETMAP_memcache_servers'"/g' $workdir/config/settings.yml - -### SETTING UP NOMINATIM URL -sed -i -e 's/nominatim.openstreetmap.org/'$NOMINATIM_URL'/g' $workdir/config/settings.yml - -#### SETTING UP OVERPASS URL -sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/config/settings.yml -sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/views/site/export.html.erb -sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/assets/javascripts/index/export.js - -### SETTING UP ORGANIZATION -sed -i -e 's/OpenStreetMap/'$ORGANIZATION_NAME'/g' $workdir/config/settings.yml -ORGANIZATION_NAME_LOWER=$(echo "$ORGANIZATION_NAME" | tr '[:upper:]' '[:lower:]') -sed -i -e 's/openstreetmap/'"$ORGANIZATION_NAME_LOWER"'/g' "$workdir/config/settings.yml" - -### ADDING DOORKEEPER_SIGNING_KEY -openssl genpkey -algorithm RSA -out private.pem -chmod 400 /var/www/private.pem -export DOORKEEPER_SIGNING_KEY=$(cat /var/www/private.pem | sed -e '1d;$d' | tr -d '\n') -sed -i "s#PRIVATE_KEY#${DOORKEEPER_SIGNING_KEY}#" $workdir/config/settings.yml - -#### CHECK IF DB IS ALREADY UP AND START THE APP -flag=true -site_loading=true - -while "$flag" = true; do - pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue - flag=false - - until $(curl -sf -o /dev/null $SERVER_URL); do - if [ "$site_loading" = true ]; then - echo "Waiting to start Rails ports server..." - site_loading=false - fi + encoding: utf8 +EOF + + ##### Setting up S3 storage + if [ "$RAILS_STORAGE_SERVICE" == "s3" ]; then + [[ -z "$RAILS_STORAGE_REGION" || -z "$RAILS_STORAGE_BUCKET" ]] && { + echo "Error: RAILS_STORAGE_REGION or RAILS_STORAGE_BUCKET not set." + exit 1 + } + + cat <> "$workdir/config/storage.yml" +s3: + service: S3 + region: '$RAILS_STORAGE_REGION' + bucket: '$RAILS_STORAGE_BUCKET' +EOF + echo "S3 storage configuration set successfully." + fi + + #### Initializing an empty $workdir/config/settings.local.yml file, typically used for development settings + echo "" > $workdir/config/settings.local.yml + + #### Setting up server_url and server_protocol + sed -i -e 's/^server_protocol: ".*"/server_protocol: "'$SERVER_PROTOCOL'"/g' $workdir/config/settings.yml + sed -i -e 's/^server_url: ".*"/server_url: "'$SERVER_URL'"/g' $workdir/config/settings.yml + + ### Setting up website status + sed -i -e 's/^status: ".*"/status: "'$WEBSITE_STATUS'"/g' $workdir/config/settings.yml + + #### Setting up mail sender + sed -i -e 's/smtp_address: ".*"/smtp_address: "'$MAILER_ADDRESS'"/g' $workdir/config/settings.yml + sed -i -e 's/smtp_port: .*/smtp_port: '$MAILER_PORT'/g' $workdir/config/settings.yml + sed -i -e 's/smtp_domain: ".*"/smtp_domain: "'$MAILER_DOMAIN'"/g' $workdir/config/settings.yml + sed -i -e 's/smtp_authentication: .*/smtp_authentication: "login"/g' $workdir/config/settings.yml + sed -i -e 's/smtp_user_name: .*/smtp_user_name: "'$MAILER_USERNAME'"/g' $workdir/config/settings.yml + sed -i -e 's/smtp_password: .*/smtp_password: "'$MAILER_PASSWORD'"/g' $workdir/config/settings.yml + + ### Setting up oauth id and key for iD editor + sed -i -e 's/^oauth_application: ".*"/oauth_application: "'$OAUTH_CLIENT_ID'"/g' $workdir/config/settings.yml + sed -i -e 's/^oauth_key: ".*"/oauth_key: "'$OAUTH_KEY'"/g' $workdir/config/settings.yml + + #### Setting up id key for the website + sed -i -e 's/^id_application: ".*"/id_application: "'$OPENSTREETMAP_id_key'"/g' $workdir/config/settings.yml + + #### Setup env vars for memcached server + sed -i -e 's/memcache_servers: \[\]/memcache_servers: "'$OPENSTREETMAP_memcache_servers'"/g' $workdir/config/settings.yml + + #### Setting up nominatim url + sed -i -e 's/nominatim-api.openstreetmap.org/'$NOMINATIM_URL'/g' $workdir/config/settings.yml + + ## Setting up overpass url + sed -i -e 's/overpass-api.openstreetmap.org/'$OVERPASS_URL'/g' $workdir/config/settings.yml + sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/views/site/export.html.erb + sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/assets/javascripts/index/export.js + + ## Setting up required credentials + echo $RAILS_CREDENTIALS_YML_ENC > config/credentials.yml.enc + echo $RAILS_MASTER_KEY > config/master.key + chmod 600 config/credentials.yml.enc config/master.key + + #### Adding doorkeeper_signing_key + openssl genpkey -algorithm RSA -out private.pem + chmod 400 /var/www/private.pem + export DOORKEEPER_SIGNING_KEY=$(cat /var/www/private.pem | sed -e '1d;$d' | tr -d '\n') + sed -i "s#PRIVATE_KEY#${DOORKEEPER_SIGNING_KEY}#" $workdir/config/settings.yml +} + +restore_db() { + export PGPASSWORD="$POSTGRES_PASSWORD" + curl -s -o backup.sql "$BACKUP_FILE_URL" || { + echo "Error: Failed to download backup file." + exit 1 + } + + psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f backup.sql && \ + echo "Database restored successfully." || \ + { echo "Database restore failed."; exit 1; } +} + +start_background_jobs() { + while true; do + pkill -f "rake jobs:work" + bundle exec rake jobs:work --trace >> "$workdir/log/jobs_work.log" 2>&1 & + echo "Restarted rake jobs at $(date)" + sleep 1h + done +} + +setup_production() { + setup_env_vars + + echo "Waiting for PostgreSQL to be ready..." + until pg_isready -h "$POSTGRES_HOST" -p 5432; do sleep 2 - done & - time rails i18n:js:export assets:precompile - bundle exec rails db:migrate - /openstreetmap-cgimap/build/openstreetmap-cgimap \ - --port=8000 \ - --daemon \ - --instances=3 \ - --dbname=$POSTGRES_DB \ - --host=$POSTGRES_HOST \ - --username=$POSTGRES_USER \ - --password=$POSTGRES_PASSWORD \ - --logfile log/cgimap.log + done + + # echo "Running asset precompilation..." + # time bundle exec rake i18n:js:export assets:precompile + + echo "Copying static assets..." + cp "$workdir/public/leaflet-ohm-timeslider-v2/assets/"* "$workdir/public/assets/" + + echo "Running database migrations..." + time bundle exec rails db:migrate + + if [ "$EXTERNAL_CGIMAP" == "false" ]; then + echo "Running cgimap..." + ./cgimap.sh + fi + + echo "Starting Apache server..." + apachectl -k start -DFOREGROUND & + start_background_jobs +} + + +setup_development() { + restore_db + cp "$workdir/config/example.storage.yml" "$workdir/config/storage.yml" + cp /tmp/settings.yml "$workdir/config/settings.yml" + setup_env_vars + bundle exec bin/yarn install + bundle exec rails db:migrate --trace bundle exec rake jobs:work & - apachectl -k start -DFOREGROUND -done + rails server --log-to-stdout +} + +####################### Setting up Development or Production mode ####################### +if [ "$ENVIRONMENT" = "development" ]; then + setup_development +else + setup_production +fi diff --git a/osm-seed/templates/cgimap/cgimap-deployment.yaml b/osm-seed/templates/cgimap/cgimap-deployment.yaml index 02cd77b8..e55d10a8 100644 --- a/osm-seed/templates/cgimap/cgimap-deployment.yaml +++ b/osm-seed/templates/cgimap/cgimap-deployment.yaml @@ -22,6 +22,7 @@ spec: release: {{ .Release.Name }} run: {{ .Release.Name }}-cgimap-deployment spec: + priorityClassName: {{ .Release.Name }}-{{ .Values.cgimap.priorityClass | default "high-priority" }} containers: - name: {{ .Chart.Name }}-cgimap image: "{{ .Values.cgimap.image.name }}:{{ .Values.cgimap.image.tag }}" @@ -61,7 +62,7 @@ spec: value: {{ .Values.db.env.POSTGRES_USER }} - name: WEBSITE_STATUS value: {{ .Values.web.env.WEBSITE_STATUS }} - - name: OPENSTREETMAP_MEMCACHE_SERVERS + - name: MEMCACHE_SERVER value: {{ .Release.Name }}-memcached:11211 {{- if .Values.cgimap.nodeSelector.enabled }} nodeSelector: diff --git a/osm-seed/templates/cgimap/cgimap-hpa.yaml b/osm-seed/templates/cgimap/cgimap-hpa.yaml index 4fd5a128..eec6b732 100644 --- a/osm-seed/templates/cgimap/cgimap-hpa.yaml +++ b/osm-seed/templates/cgimap/cgimap-hpa.yaml @@ -23,4 +23,4 @@ spec: target: type: Utilization averageUtilization: {{ .Values.cgimap.autoscaling.memoryUtilization }} -{{- end }} \ No newline at end of file +{{- end }} From 347cec42d1308cead5cbf828b4c946d4209c213b Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 16 Apr 2025 12:35:15 -0500 Subject: [PATCH 21/25] Fix apache config --- images/web/config/production.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 718c4779..e3d1f3da 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -12,10 +12,10 @@ RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - ; # Redirect to www openstreetmap.org - ; RewriteCond %{HTTP_HOST} =openstreetmap.org - ; RewriteCond %{HTTP_HOST} !^www\. [NC] - ; RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + # Redirect to www openstreetmap.org + # RewriteCond %{HTTP_HOST} =openstreetmap.org + # RewriteCond %{HTTP_HOST} !^www\. [NC] + # RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] CGIPassAuth On From 770ca83bfe9fd655ea99a648b0b91f6dbaf920ef Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 18 Apr 2025 08:14:14 -0500 Subject: [PATCH 22/25] Add PG password for tiler-db config as env var --- osm-seed/templates/tiler-db/tiler-db-configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/osm-seed/templates/tiler-db/tiler-db-configmap.yaml b/osm-seed/templates/tiler-db/tiler-db-configmap.yaml index 7072e059..784d0cd0 100644 --- a/osm-seed/templates/tiler-db/tiler-db-configmap.yaml +++ b/osm-seed/templates/tiler-db/tiler-db-configmap.yaml @@ -12,6 +12,7 @@ data: {{- end }} POSTGRES_DB: {{ .Values.tilerDb.env.POSTGRES_DB }} POSTGRES_PASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }} + PGPASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }} POSTGRES_USER: {{ .Values.tilerDb.env.POSTGRES_USER | quote }} POSTGRES_PORT: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }} {{- end }} From 943eb10faaac9cdf2b18ead8fb98cb1148a38d90 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 22 Apr 2025 09:54:35 -0500 Subject: [PATCH 23/25] Update cgimap dockerfile using code form original repo --- images/cgimap/Dockerfile | 64 +++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/images/cgimap/Dockerfile b/images/cgimap/Dockerfile index 660cfec0..51079999 100644 --- a/images/cgimap/Dockerfile +++ b/images/cgimap/Dockerfile @@ -1,41 +1,45 @@ -FROM ruby:3.3.0 AS builder +FROM debian:bookworm-slim AS builder ENV DEBIAN_FRONTEND=noninteractive -ENV workdir=/var/www + +RUN apt-get update -qq && \ + apt-get install -y gcc g++ make cmake \ + libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \ + libboost-program-options-dev libcrypto++-dev libyajl-dev \ + libpqxx-dev zlib1g-dev libfmt-dev \ + postgresql-15 postgresql-server-dev-all dpkg-dev file ca-certificates git \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Clone application ENV CGIMAP_GITSHA=8ea707e10aeab5698e6859856111816d75354592 -ENV cgimap=/tmp/openstreetmap-cgimap +RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app \ +&& git checkout $CGIMAP_GITSHA -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential cmake git curl file \ - libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \ - libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \ - libargon2-dev libyajl-dev \ - && rm -rf /var/lib/apt/lists/* +# Compile, install and remove source +RUN mkdir build && cd build && \ + CXXFLAGS="-flto=auto -ffat-lto-objects -flto=auto -ffat-lto-objects -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release && \ + make -j${nproc} && \ + ctest --output-on-failure && \ + cmake --build . -t package -RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ - && cd $cgimap \ - && git checkout $CGIMAP_GITSHA \ - && mkdir build && cd build && cmake .. && cmake --build . +FROM debian:bookworm-slim +COPY --from=builder /app/build/*.deb /app_deb/ -FROM ruby:3.3.0-slim +RUN apt-get update -qq && \ + apt install --no-install-recommends -y /app_deb/*.deb postgresql-client procps && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -ENV DEBIAN_FRONTEND=noninteractive +RUN groupadd -g 61000 cgimap && \ + useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap + +EXPOSE 8000 -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl gnupg && \ - echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ - curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - libxml2 libpqxx-6.4 libfcgi zlib1g libbrotli1 \ - libboost-program-options1.74.0 libfmt-dev libmemcached11 libcrypto++8 \ - libargon2-1 libyajl2 libapache2-mod-fcgid postgresql-client \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/* - -COPY --from=builder /tmp/openstreetmap-cgimap/build/openstreetmap-cgimap /usr/local/bin/openstreetmap-cgimap - -COPY *.sh / -RUN chmod +x /*.sh && echo "/usr/local/lib" > /etc/ld.so.conf.d/local_libs.conf && ldconfig +COPY ./*.sh . CMD ["/start.sh"] From 31aa902eab5012e71c1f0bd075cb29891fae2284 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 22 Apr 2025 13:12:31 -0500 Subject: [PATCH 24/25] Enable resources option for requests and limits - cgimap --- osm-seed/templates/cgimap/cgimap-deployment.yaml | 4 ++++ osm-seed/values.yaml | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/osm-seed/templates/cgimap/cgimap-deployment.yaml b/osm-seed/templates/cgimap/cgimap-deployment.yaml index e55d10a8..9ce70584 100644 --- a/osm-seed/templates/cgimap/cgimap-deployment.yaml +++ b/osm-seed/templates/cgimap/cgimap-deployment.yaml @@ -42,12 +42,16 @@ spec: failureThreshold: 3 {{- if .Values.cgimap.resources.enabled }} resources: + {{- if .Values.cgimap.resources.requests.enabled }} requests: memory: {{ .Values.cgimap.resources.requests.memory }} cpu: {{ .Values.cgimap.resources.requests.cpu }} + {{- end }} + {{- if .Values.cgimap.resources.limits.enabled }} limits: memory: {{ .Values.cgimap.resources.limits.memory }} cpu: {{ .Values.cgimap.resources.limits.cpu }} + {{- end }} {{- end }} env: - name: POSTGRES_HOST diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 14a3d8a0..34b0aa19 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -214,11 +214,13 @@ cgimap: resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + enabled: false + memory: "1Gi" + cpu: "2" limits: - memory: '24Gi' - cpu: '10' + enabled: false + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false autoscaling: From 15e362aa95df1c64772d21ac9a407f3573ddc41f Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 22 Apr 2025 18:31:29 -0500 Subject: [PATCH 25/25] Evaluate memoryUtilization for web container - autoscaling --- osm-seed/templates/web/web-hpa.yaml | 10 +++++++--- osm-seed/values.yaml | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/osm-seed/templates/web/web-hpa.yaml b/osm-seed/templates/web/web-hpa.yaml index 30e4a267..e1df7d14 100644 --- a/osm-seed/templates/web/web-hpa.yaml +++ b/osm-seed/templates/web/web-hpa.yaml @@ -1,5 +1,4 @@ -{{- if .Values.web.enabled -}} -{{- if .Values.web.autoscaling.enabled -}} +{{- if and .Values.web.enabled .Values.web.autoscaling.enabled -}} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: @@ -18,5 +17,10 @@ spec: target: type: Utilization averageUtilization: {{ .Values.web.autoscaling.cpuUtilization }} -{{- end }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.web.autoscaling.memoryUtilization }} {{- end }} diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 34b0aa19..1f16c379 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -178,7 +178,8 @@ web: enabled: false minReplicas: 1 maxReplicas: 3 - cpuUtilization: 80 + cpuUtilization: 60 + memoryUtilization: 60 sharedMemorySize: 256Mi livenessProbeExec: true # ====================================================================================================