Skip to content

Commit 1235354

Browse files
Rub21Rub21
authored
Tiler software updates (#335)
* Update Tiler containers * Update config for tiler server * Split provider layers * Update docker file - tiler server * Update providers * Update ne layers - tiler server * Update container for tiler
1 parent 1719642 commit 1235354

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3535
-449
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,5 @@ overpass-api-db/
3636
data/*/
3737
!data/README.md
3838
images/changeset-replication-job/config.yaml
39-
values_copy.yaml
39+
values_copy.yaml
40+
images/tiler-server/tegola-cache/

‎compose/tiler.yml

+38-46
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: '3.8'
1+
# version: '3.8'
22
services:
33
# ######################################################
44
# ### Tiler DB
@@ -8,61 +8,53 @@ services:
88
build:
99
context: ../images/tiler-db
1010
dockerfile: Dockerfile
11-
# ports:
12-
# - "5433:5432"
11+
ports:
12+
- "5432:5432"
1313
volumes:
1414
- ../data/tiler-db-data:/var/lib/postgresql/data
1515
env_file:
1616
- ../envs/.env.tiler-db
1717
restart: always
18-
# ######################################################
19-
# ### Tiler imposm for importing data
20-
# ######################################################
21-
tiler-imposm:
22-
image: osmseed-tiler-imposm:v1
23-
build:
24-
context: ../images/tiler-imposm
25-
dockerfile: Dockerfile
26-
volumes:
27-
- ../data/tiler-imposm-data:/mnt/data
28-
# - ../images/tiler-imposm:/osm
29-
depends_on:
30-
- tiler-db
31-
command:
32-
- sh
33-
- -c
34-
- "sleep 60 && ./start.sh"
35-
env_file:
36-
- ../envs/.env.tiler-db
37-
- ../envs/.env.tiler-imposm
38-
healthcheck:
39-
test: ["CMD", "./liveness.sh"]
40-
interval: 30s
41-
timeout: 10s
42-
retries: 3
43-
restart: always
44-
# ######################################################
45-
# ### Tiler server
46-
# ######################################################
18+
# # ######################################################
19+
# # ### Tiler imposm for importing data
20+
# # ######################################################
21+
# tiler-imposm:
22+
# image: osmseed-tiler-imposm:v1
23+
# build:
24+
# context: ../images/tiler-imposm
25+
# dockerfile: Dockerfile
26+
# volumes:
27+
# - ../data/tiler-imposm-data:/mnt/data
28+
# # - ../images/tiler-imposm:/osm
29+
# command:
30+
# - sh
31+
# - -c
32+
# - "./start.sh"
33+
# env_file:
34+
# - ../envs/.env.tiler-db
35+
# - ../envs/.env.tiler-imposm
36+
# healthcheck:
37+
# test: ["CMD", "./liveness.sh"]
38+
# interval: 30s
39+
# timeout: 10s
40+
# retries: 3
41+
# restart: always
42+
# # ######################################################
43+
# # ### Tiler server
44+
# # ######################################################
4745
tiler-server:
4846
image: osmseed-tiler-server:v1
47+
# platform: linux/amd64
4948
build:
5049
context: ../images/tiler-server
5150
dockerfile: Dockerfile
5251
volumes:
53-
- ../data/tiler-server-data:/mnt/data
54-
- ../data/tiler-imposm-data:/mnt/data/imposm
55-
depends_on:
56-
- tiler-db
57-
- tiler-imposm
52+
- ../images/tiler-server:/app
53+
- ../data/tiler-imposm-data:/mnt/data/imposm
5854
ports:
59-
- "9090:9090"
60-
command:
61-
- sh
62-
- -c
63-
- "./start.sh & ./expire-watcher.sh"
55+
- "9090:9090"
56+
command: "./start.sh"
6457
env_file:
65-
- ../envs/.env.tiler-db
66-
- ../envs/.env.tiler-server
67-
restart: always
68-
58+
- ../envs/.env.tiler-db
59+
- ../envs/.env.tiler-server
60+
restart: always

‎envs/.env.tiler-imposm.example

+3-1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,6 @@ TILER_IMPORT_FROM=osm
77
TILER_IMPORT_PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
88
SEQUENCE_NUMBER=4768
99
REPLICATION_URL=https://download.openstreetmap.fr/replication/europe/monaco/minute/
10-
OVERWRITE_STATE=false
10+
OVERWRITE_STATE=false
11+
IMPORT_NATURAL_EARTH=true
12+
IMPORT_OSM_LAND=true

‎images/tiler-db/Dockerfile

+4-8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,7 @@
1-
FROM mdillon/postgis:9.5
2-
RUN rm /etc/apt/sources.list.d/pgdg.list
3-
RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
4-
RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list
5-
RUN sed -i '/stretch-updates/d' /etc/apt/sources.list
6-
RUN apt-get update && apt-get -y install apt-transport-https
7-
RUN echo "deb https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 9.5" /etc/apt/sources.list.d/pgdg.list
8-
RUN apt-get update && apt-get install -y git ca-certificates
1+
FROM postgis/postgis:14-3.4
2+
RUN apt-get update
3+
RUN apt-get install -y git ca-certificates
4+
95
COPY ./config/docker-entrypoint.sh /usr/local/bin/
106
RUN mkdir -p /docker-entrypoint-initdb.d
117
COPY ./config/initdb_db.sh /docker-entrypoint-initdb.d/postgis.sh

‎images/tiler-db/config/initdb_db.sh

+7-17
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,10 @@
11
#!/bin/sh
22
set -e
33

4-
# Create the 'template_postgis' template db
5-
"${psql[@]}" <<- 'EOSQL'
6-
CREATE DATABASE template_postgis;
7-
UPDATE pg_database SET datistemplate = TRUE WHERE datname = 'template_postgis';
8-
EOSQL
9-
10-
# Load PostGIS into both template_database and $POSTGRES_DB
11-
for DB in template_postgis "$POSTGRES_DB"; do
12-
echo "Loading PostGIS extensions into $DB"
13-
"${psql[@]}" --dbname="$DB" <<-'EOSQL'
14-
CREATE EXTENSION IF NOT EXISTS postgis;
15-
CREATE EXTENSION IF NOT EXISTS postgis_topology;
16-
CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
17-
CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
18-
CREATE EXTENSION IF NOT EXISTS hstore;
19-
EOSQL
20-
done
4+
# Add hstore into the DB
5+
for DB in template_postgis "$POSTGRES_DB" "${@}"; do
6+
echo "Updating extensions '$DB'"
7+
psql --dbname="$DB" -c "
8+
CREATE EXTENSION IF NOT EXISTS hstore;
9+
"
10+
done

‎images/tiler-server/Dockerfile

+22-38
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,41 @@
1-
FROM golang:1.14.1-alpine3.11 AS build
2-
ENV VERSION="v0.8.1"
3-
RUN apk update
4-
RUN apk add musl-dev=1.1.24-r3 \
5-
gcc \
6-
bash \
7-
git \
8-
postgresql \
9-
postgresql-contrib
1+
FROM gospatial/tegola:v0.20.0
102

11-
RUN mkdir -p /go/src/github.com/go-spatial/tegola
12-
RUN git clone https://github.com/go-spatial/tegola.git /go/src/github.com/go-spatial/tegola
13-
RUN cd /go/src/github.com/go-spatial/tegola && git checkout v0.13.0
14-
RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \
15-
&& go build -gcflags "-N -l" -o /opt/tegola \
16-
&& chmod a+x /opt/tegola
17-
RUN ln -s /opt/tegola /usr/bin/tegola
18-
19-
RUN apk add --update \
20-
python3 \
21-
py-pip \
22-
py-cffi \
23-
py-cryptography \
24-
&& pip install --upgrade pip \
25-
&& apk add --virtual build-deps \
3+
RUN apk --no-cache add \
264
gcc \
275
libffi-dev \
28-
python3-dev \
6+
python3 \
7+
py3-pip \
8+
py3-cffi \
9+
py3-cryptography \
2910
linux-headers \
3011
musl-dev \
3112
openssl-dev \
32-
curl
13+
bash \
14+
git \
15+
postgresql-client \
16+
coreutils \
17+
jq
3318

34-
# Install aws-cli and gsutil
35-
RUN pip3 install awscli
36-
RUN curl -sSL https://sdk.cloud.google.com | bash
37-
RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
19+
RUN pip install --upgrade pip && pip3 install awscli mercantile
3820

39-
RUN pip3 install mercantile \
40-
&& apk del build-deps \
41-
&& rm -rf /var/cache/apk/* \
42-
&& apk --purge -v del py-pip
4321

44-
RUN apk add --update coreutils jq && rm -rf /var/cache/apk/*
22+
RUN ln -s /opt/tegola /usr/bin/tegola
23+
24+
COPY ./config /opt/config/
25+
COPY build_config.py /opt/
26+
RUN mkdir /opt/tegola_config
27+
RUN cd /opt/ && python build_config.py \
28+
--template config/config.template.toml \
29+
--providers config/providers \
30+
--output /opt/tegola_config/config.toml
4531

46-
# Volumen
47-
VOLUME /mnt/data
4832
# Copy config and exec files
49-
COPY ./config/config.toml /opt/tegola_config/config.toml
5033
COPY ./tile2bounds.py .
5134
COPY ./start.sh .
5235
COPY ./expire-watcher.sh .
5336
COPY ./seed-by-diffs.sh .
5437
COPY ./tile_cache_downloader.sh .
5538
COPY ./rm_tegola_ps.sh .
5639

40+
ENTRYPOINT ["/bin/bash", "-c"]
5741
CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh

‎images/tiler-server/build_config.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import os
2+
import argparse
3+
parser = argparse.ArgumentParser(description='Merge TOML files into a configuration file.')
4+
parser.add_argument('--template', default='config/config.template.toml', help='Path to the configuration template file.')
5+
parser.add_argument('--providers', default='config/providers', help='Directory containing provider TOML files.')
6+
parser.add_argument('--output', default='config/config.toml', help='Output configuration file path.')
7+
args = parser.parse_args()
8+
9+
config_template_file = args.template
10+
providers_dir = args.providers
11+
output_file_path = args.output
12+
toml_files = [file for file in os.listdir(providers_dir) if file.endswith(".toml")]
13+
14+
# Read TOML files
15+
new_configs = {}
16+
for toml_file in toml_files:
17+
dir_toml_file = os.path.join(providers_dir, toml_file)
18+
with open(dir_toml_file, "r") as file:
19+
new_configs[dir_toml_file] = file.read()
20+
21+
with open(config_template_file, "r") as main_file:
22+
content = main_file.read()
23+
24+
# Replace the content of main.toml with the content read from other TOML files
25+
for toml_file, toml_file_content in new_configs.items():
26+
print(f"Copy {toml_file} to config.toml")
27+
section_header = "[['{}']]".format(toml_file.replace("config/", ""))
28+
indentation_level = content.find(section_header)
29+
if indentation_level != -1:
30+
# Find the appropriate number of tabs or spaces for indentation
31+
preceding_newline = content.rfind('\n', 0, indentation_level)
32+
indentation = content[preceding_newline + 1:indentation_level]
33+
toml_file_content = f"###### From {toml_file} \n" + toml_file_content
34+
new_values=toml_file_content.replace("\n", "\n" + indentation)
35+
content = content.replace(section_header, new_values)
36+
37+
with open(output_file_path, "w") as output_file:
38+
output_file.write(content)

0 commit comments

Comments
 (0)