Update web container and TM config (#315)

* Update config for TM to use a db

* Update web container

* Update config to set up web and tm

* Update env vars for web and tm

* Restart containers

* Update port for planet api

* Update tiler containers

* Update config

* Update compose config

* Delete tasking manager web

* Update website version

* Update config

* Generate doorkeeper_signing_key and set in config

* Remove upload to s3 the logs from restore db

* Add env var to change org name

* Disable cgimap

* Update the private key creation

* Comment cgimap cli

* Update private key creation

* Update website version

* Update api file service

* Update api db

Author: Rub21

compose/nominatim.yml

@@ -10,9 +10,10 @@ services:
       dockerfile: Dockerfile
     ports:
       - '8080:8080'
-      - '5432:5432'
+    #   - '5432:5432'
     volumes:
       - ../data/nominatim-data:/var/lib/postgresql/14/main
       # - ../../Nominatim/nominatim/tokenizer/sanitizers:/usr/local/lib/nominatim/lib-python/nominatim/tokenizer/sanitizers
     env_file:
       - ../envs/.env.nominatim
+    restart: always

compose/planet.yml

@@ -16,6 +16,7 @@ services:
     - ../envs/.env.db
     - ../envs/.env.db-utils
     - ../envs/.env.cloudprovider
+    restart: always
 
   ######################################################
   ### Planet replication section
@@ -32,6 +33,7 @@ services:
     - ../envs/.env.db
     - ../envs/.env.db-utils
     - ../envs/.env.cloudprovider
+    restart: always
 
   #####################################################
   ## OSM full planet replication
@@ -48,6 +50,7 @@ services:
     - ../envs/.env.db
     - ../envs/.env.db-utils
     - ../envs/.env.cloudprovider
+    restart: always
 
   #####################################################
   ## Changeset replications
@@ -65,6 +68,7 @@ services:
     - ../envs/.env.db
     - ../envs/.env.db-utils
     - ../envs/.env.cloudprovider
+    restart: always
 
   ######################################################
   ### NGINX container for serving files
@@ -76,7 +80,7 @@ services:
       dockerfile: Dockerfile
     ports:
     - "8081:80"
-    - "3000:3000"
+    - "8082:3000"
     volumes:
     # Serve minute replication files
     - ../data/replication-job-data:/usr/share/nginx/html/server/static-files/replication/minute
@@ -87,3 +91,7 @@ services:
     - '../data/changeset-replication-job-data:/usr/share/nginx/html/server/static-files/changesets'
     # Development mode
     # - ./../images/planet-files/:/usr/share/nginx/html/
+    restart: always
+networks:
+  osm-seed:
+    driver: bridge

compose/tasking-manager.yml

@@ -1,16 +1,33 @@
 version: '3'
 services:
-  ######################################################
-  ### Tasking Manager section
-  ######################################################
-  tasking-manager-api:
+  tmdb:
+    image: mdillon/postgis:11
+    # ports:
+    # - '5432:5432'
+    volumes:
+    - ../data/tasking-manager-db:/db
+    env_file:
+    - ../envs/.env.tasking-manager-db
+    restart: always
+  migration:
     image: osmseed-tasking-manager-api:v1
     build:
       context: ../images/tasking-manager-api
       dockerfile: Dockerfile
+    restart: on-failure
+    command: >
+      sh -c "sleep 10s && python3 manage.py db upgrade"
+    env_file:
+    - ../envs/.env.tasking-manager
+    - ../envs/.env.tasking-manager-db
+  tmapi:
+    build:
+      context: ../images/tasking-manager-api
+      dockerfile: Dockerfile
+    image: osmseed-tasking-manager-api:v1
     ports:
-      - '5000:5000'
-    volumes:
-      - ../data/tasking-manager-api-db:/db
+    - '5000:5000'
     env_file:
-      - ../envs/.env.tasking-manager
+    - ../envs/.env.tasking-manager
+    - ../envs/.env.tasking-manager-db
+    restart: always

compose/tiler.yml

@@ -8,8 +8,8 @@ services:
     build:
       context: ../images/tiler-db
       dockerfile: Dockerfile
-    ports:
-    - "5433:5432"
+    # ports:
+    # - "5433:5432"
     volumes:
     - ../data/tiler-db-data:/var/lib/postgresql/data
     env_file:
@@ -42,7 +42,7 @@ services:
       retries: 3
     restart: always
   # ######################################################
-  # ### Tiler imposm for importing data
+  # ### Tiler server
   # ######################################################
   tiler-server:
     image: osmseed-tiler-server:v1
@@ -51,6 +51,7 @@ services:
       dockerfile: Dockerfile
     volumes:
     - ../data/tiler-server-data:/mnt/data
+    - ../data/tiler-imposm-data:/mnt/data/imposm
     depends_on:
     - tiler-db
     - tiler-imposm
@@ -59,8 +60,9 @@ services:
     command:
     - sh
     - -c
-    - "sleep 60 && ./start.sh"
+    - "./start.sh & ./expire-watcher.sh"
     env_file:
     - ../envs/.env.tiler-db
     - ../envs/.env.tiler-server
     restart: always
+  

compose/web.yml

@@ -9,12 +9,13 @@ services:
     build:
       context: ../images/db
       dockerfile: Dockerfile
-    ports:
-    - '5432:5432'
-    # volumes:
-    #   - ../data/db-data:/var/lib/postgresql/data
+    # ports:
+    # - '5432:5432'
+    volumes:
+    - ../data/db-data-pg14:/var/lib/postgresql/data
     env_file:
     - ../envs/.env.db
+    restart: always
   web:
     image: osmseed-web:v1
     build:
@@ -27,3 +28,9 @@ services:
     - ../envs/.env.db
     depends_on:
     - db
+    restart: always
+  memcache:
+    image: memcached:latest
+    ports:
+    - '11211:11211'
+    restart: always

envs/.env.tasking-manager-db.example

@@ -0,0 +1,4 @@
+POSTGRES_DB=tm
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=1234
+POSTGRES_PORT=5432

envs/.env.tasking-manager.example

@@ -37,20 +37,15 @@ POTLATCH2_EDITOR_URL=https://www.openstreetmap.org/edit?editor=potlatch2
 #
 TM_SECRET=xyz
 
-# OpenStreetMap OAuth consumer key and secret (required)
-#
+# OpenStreetMap OAuth 1 consumer key and secret (required)
+
 TM_CONSUMER_KEY=foo
 TM_CONSUMER_SECRET=xyz
 
 TM_DEFAULT_CHANGESET_COMMENT="#{osmseed}-project"
 
 # The connection to the postgres database (required)
-#
-POSTGRES_DB=tm
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=tm
-POSTGRES_ENDPOINT=localhost
-POSTGRES_PORT=5432
+POSTGRES_HOST=tmdb
 
 # The address to use as the sender on auto generated emails (optional)
 #

envs/.env.web.example

@@ -20,16 +20,27 @@ [email protected]
 MAILER_PASSWORD=abc...
 [email protected]
 MAILER_PORT=25
+OSM_memcache_servers=""
 
 # Nominatim settings
 NOMINATIM_URL=nominatim-api
 
 # Make sure that the first time this (OPENSTREETMAP_id_key) value is empty. 
-# Once you start you server and create your OAuth 1, fill the value with the Consumer Key.
-OPENSTREETMAP_id_key=""
-OSM_memcache_servers=""
+# Once you start you server and create your OAuth 2.
+### Create OAuth 2 applications
+### Client ID=OPENSTREETMAP_id_key
+OPENSTREETMAP_id_key="xyz..."
+
+### OpenStreetMap Web Site
+# Client ID=OAUTH_CLIENT_ID
+# Client Secret=OAUTH_KEY	
+OAUTH_CLIENT_ID="abc..."
+OAUTH_KEY="efg..."
 
 # NEW_RELIC settings
 # TODO: Needs to be implemented the New Relic functions in order to make running in the web container
 NEW_RELIC_LICENSE_KEY="..."
 NEW_RELIC_APP_NAME="..."
+
+### Set organization name, by default : OpenStreetMap
+ORGANIZATION_NAME=OSMSeed

images/backup-restore/start.sh

@@ -63,7 +63,7 @@ restoreDB() {
 		wget -O ${RESTORE_FILE} ${RESTORE_URL_FILE}
 		echo "Restoring ${RESTORE_URL_FILE} in ${POSTGRES_DB}"
 		gunzip -c <${RESTORE_FILE} | psql -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} | tee ${LOG_RESULT_FILE}
-		aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE}
+		# aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE}
 		echo "Import data to ${POSTGRES_DB} has finished ..."
 	done
 }

images/db/Dockerfile

@@ -1,25 +1,18 @@
-FROM postgres:11
-RUN rm /etc/apt/sources.list.d/pgdg.list
-RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
-RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list 
-RUN sed -i '/stretch-updates/d' /etc/apt/sources.list 
-RUN apt-get update && apt-get -y install apt-transport-https
-RUN echo "deb [ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 11" >/etc/apt/sources.list.d/pgdg.list
+FROM postgres:14
 RUN apt-get update \
     && apt-get install -y \
-    postgresql-server-dev-11 \
+    postgresql-server-dev-14 \
     make \
     build-essential \
-    postgresql-11-postgis-2.5 \
+    postgresql-14-postgis-3 \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
-    
+
 ADD functions/functions.sql /usr/local/share/osm-db-functions.sql
 ADD docker_postgres.sh /docker-entrypoint-initdb.d/
 RUN mkdir -p db
 RUN mkdir -p lib
 ADD functions/ db/functions/
 ADD lib/quad_tile/ lib/quad_tile/
-
 RUN make -C db/functions/
 RUN chown -R postgres lib/
-RUN chown -R postgres db/
+RUN chown -R postgres db/

images/planet-files/index.html

@@ -22,7 +22,7 @@
     <script>
       // const API_FILES = "http://localhost:3000";
       const HOST_NAME= window.location.hostname
-      const API_FILES = `http://${HOST_NAME}:3000`;
+      const API_FILES = "API_FILES_SERVICES";
       document.title = HOST_NAME;
       async function fetchFiles(directory) {
         const response = await fetch(`${API_FILES}/${directory}`);

images/tiler-server/Dockerfile

@@ -10,7 +10,7 @@ RUN apk add musl-dev=1.1.24-r3 \
 
 RUN mkdir -p /go/src/github.com/go-spatial/tegola
 RUN git clone https://github.com/go-spatial/tegola.git /go/src/github.com/go-spatial/tegola
-RUN cd /go/src/github.com/go-spatial/tegola && git checkout v0.8.1
+RUN cd /go/src/github.com/go-spatial/tegola && git checkout v0.13.0
 RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \
 	&& go build -gcflags "-N -l" -o /opt/tegola \ 
 	&& chmod a+x /opt/tegola
@@ -36,7 +36,7 @@ RUN pip3 install awscli
 RUN curl -sSL https://sdk.cloud.google.com | bash
 RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
 
-RUN pip install mercantile \
+RUN pip3 install mercantile \
     && apk del build-deps \
     && rm -rf /var/cache/apk/* \
     && apk --purge -v del py-pip

images/tiler-server/expire-watcher.sh

@@ -10,6 +10,6 @@ while true ; do
   if [ "$sum" != "$new_sum" ]; then
     ./seed-by-diffs.sh
   else
-    sleep 1
+    sleep 30s
   fi
 done

images/tiler-server/seed-by-diffs.sh

@@ -16,49 +16,32 @@ completed_jobs=$workDir/imposm/completed.list
 completed_dir=$workDir/imposm/imposm3_expire_purged
 mkdir -p $completed_dir
 
-
 # List files in expire_dir
-imp_list=`find $expire_dir -name '*.tiles' -type f`
+imp_list=$(find $expire_dir -name '*.tiles' -type f)
 
 for f in $imp_list; do
-    echo "$f" >> $queued_jobs
+    echo "$f" >>$queued_jobs
 done
 
 # Sort the files and set unique rows
-if [ -f $queued_jobs ] ; then
-    sort -u $queued_jobs > $workDir/imposm/tmp.list && mv $workDir/imposm/tmp.list $queued_jobs
+if [ -f $queued_jobs ]; then
+    sort -u $queued_jobs >$workDir/imposm/tmp.list && mv $workDir/imposm/tmp.list $queued_jobs
 fi
 
 for f in $imp_list; do
-    echo "seeding from $f"
-    # Read each line on the tiles file
-    while IFS= read -r tile
-    do
-        bounds="$(python tile2bounds.py $tile)"
-        echo tegola cache purge \
-        --config=/opt/tegola_config/config.toml \
-        --min-zoom=0 --max-zoom=20 \
-        --bounds=$bounds \
-        tile-name=$tile
-        
-        tegola cache purge \
+    # echo "Purge tiles from...$f"
+    tegola cache purge tile-list $f \
         --config=/opt/tegola_config/config.toml \
-        --min-zoom=0 --max-zoom=20 \
-        --bounds=$bounds \
-        tile-name=$tile
-        err=$?
-        if [[ $err != "0" ]]; then
-            #error
-            echo "tegola exited with error code $err"
-            # rm $queued_jobs
-            exit
-        fi
-    done < "$f"
-    echo "$f" >> $completed_jobs
+        --format="/zxy" \
+        --min-zoom=0 \
+        --max-zoom=20 \
+        --overwrite=true &
+    sleep 10s
+    echo "$f" >>$completed_jobs
     mv $f $completed_dir
 done
 
-if [ -f $queued_jobs ] ; then
-    echo "finished seeding"
+if [ -f $queued_jobs ]; then
+    # Completed queued jobs
     rm $queued_jobs
 fi