diff --git a/images/replication-job/start.sh b/images/replication-job/start.sh index 2e193768..df6df5bc 100755 --- a/images/replication-job/start.sh +++ b/images/replication-job/start.sh @@ -100,10 +100,16 @@ function enable_osmdbt_replication() { # Use osmdbt-enable-replication to set up replication properly echo "$(date +%F_%H:%M:%S): Running osmdbt-enable-replication..." - if /osmdbt/build/src/osmdbt-enable-replication -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-enable-replication.log"; then + local log_file="${logDirectory}/osmdbt-enable-replication.log" + if /osmdbt/build/src/osmdbt-enable-replication -c "$osmdbtConfig" 2>&1 | tee -a "$log_file"; then echo "$(date +%F_%H:%M:%S): Successfully enabled osmdbt replication." return 0 else + # Check if error is "already exists" - this is acceptable + if grep -qi "already exists" "$log_file" 2>/dev/null; then + echo "$(date +%F_%H:%M:%S): Replication slot '$REPLICATION_SLOT' already exists. Replication should be enabled." + return 0 + fi local error_msg="ERROR: Failed to enable osmdbt replication. Check PostgreSQL configuration (wal_level=logical, max_replication_slots >= 1, user with REPLICATION attribute)." echo "$(date +%F_%H:%M:%S): $error_msg" send_slack_message "🚨 ${ENVIROMENT:-production}: $error_msg" diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 0927843b..c039b3dc 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -1,7 +1,12 @@ FROM ruby:3.3-slim AS builder ENV DEBIAN_FRONTEND=noninteractive \ - workdir=/var/www + workdir=/var/www \ + BUNDLE_PATH=/usr/local/bundle \ + GEM_HOME=/usr/local/bundle \ + GEM_PATH=/usr/local/bundle \ + PATH="/usr/local/bundle/bin:$PATH" \ + RAILS_ENV=production WORKDIR $workdir @@ -10,25 +15,24 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ git curl gnupg build-essential \ libarchive-dev zlib1g-dev libcurl4-openssl-dev \ - apache2 apache2-dev libapache2-mod-passenger libapache2-mod-fcgid libapr1-dev libaprutil1-dev \ + apache2 apache2-dev libapache2-mod-fcgid libapr1-dev libaprutil1-dev \ postgresql-client libpq-dev libxml2-dev libyaml-dev \ - pngcrush optipng advancecomp pngquant jhead jpegoptim gifsicle libjpeg-progs \ - && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + libgd-dev \ + pngcrush optipng advancecomp pngquant jhead jpegoptim gifsicle libjpeg-progs unzip\ + && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ && apt-get install -y nodejs \ && npm install -g yarn svgo \ && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN a2enmod passenger # Clone OSM Website -ENV OPENSTREETMAP_WEBSITE_GITSHA=ea3760f94d9d74d3aaa8492182b9e1a15ec1effa -RUN rm -rf $workdir/* && \ - git clone https://github.com/openstreetmap/openstreetmap-website.git $workdir && \ - cd $workdir && \ - git checkout $OPENSTREETMAP_WEBSITE_GITSHA && \ - git fetch && rm -rf .git - -# Install Ruby/Node dependencies +ENV OPENSTREETMAP_WEBSITE_GITSHA=a244e419719ded592fb87e7ffd360f6e462a0d67 +ENV OSM_WEBSITE_URL=https://github.com/openstreetmap/openstreetmap-website/archive/${OPENSTREETMAP_WEBSITE_GITSHA}.zip +RUN rm -rf $workdir/* && curl -fsSL $OSM_WEBSITE_URL -o /tmp/openstreetmap-website.zip && \ + unzip /tmp/openstreetmap-website.zip -d /tmp && \ + mv /tmp/openstreetmap-website-$OPENSTREETMAP_WEBSITE_GITSHA/* $workdir && \ + rm -rf /tmp/* + RUN gem install bundler && \ bundle install && \ yarn install && \ @@ -45,8 +49,8 @@ RUN rm -f config/credentials.yml.enc && \ export RAILS_MASTER_KEY=$(openssl rand -hex 16) && \ export SECRET_KEY_BASE=$(bundle exec rails secret) && \ echo $RAILS_MASTER_KEY > config/master.key && \ - EDITOR="echo" RAILS_MASTER_KEY=$RAILS_MASTER_KEY rails credentials:edit && \ - RAILS_MASTER_KEY=$RAILS_MASTER_KEY rails runner "\ + EDITOR="echo" RAILS_MASTER_KEY=$RAILS_MASTER_KEY bundle exec rails credentials:edit && \ + RAILS_MASTER_KEY=$RAILS_MASTER_KEY bundle exec rails runner "\ require 'active_support/encrypted_configuration'; \ require 'yaml'; \ creds = ActiveSupport::EncryptedConfiguration.new(\ @@ -59,40 +63,103 @@ RUN rm -f config/credentials.yml.enc && \ creds.write(credentials.to_yaml); \ puts 'Credentials configured correctly.'" -# Precompile assets -RUN bundle exec rake i18n:js:export && \ - bundle exec rake assets:precompile +# Precompiling assets for production without requiring secret RAILS_MASTER_KEY +RUN SECRET_KEY_BASE_DUMMY=1 \ + bundle exec i18n export && \ + bundle exec rails assets:precompile + FROM ruby:3.3-slim ENV DEBIAN_FRONTEND=noninteractive \ - workdir=/var/www - -WORKDIR $workdir - -# Install only runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - apache2 libapache2-mod-passenger libapache2-mod-fcgid \ - libpq5 libxml2 libyaml-0-2 libarchive13 file libgd-dev \ - postgresql-client curl \ + workdir=/var/www \ + BUNDLE_PATH=/usr/local/bundle \ + GEM_HOME=/usr/local/bundle \ + GEM_PATH=/usr/local/bundle \ + PATH="/usr/local/bundle/bin:$PATH" \ + RAILS_ENV=production \ + PATH="$PATH:$GEM_HOME/bin" + +# Install base dependencies for Passenger gem compilation and runtime +RUN BUILD_DEPS=" \ + build-essential \ + apache2-dev \ + libcurl4-openssl-dev \ + zlib1g-dev \ + libssl-dev \ + npm \ + " \ + && apt-get update && apt-get install -y --no-install-recommends \ + $BUILD_DEPS \ + libgd-dev \ + apache2 \ + libapache2-mod-fcgid \ + libpq5 \ + libxml2 \ + libyaml-0-2 \ + libarchive13 \ + file \ + pngcrush \ + optipng \ + advancecomp \ + pngquant \ + jhead \ + jpegoptim \ + gifsicle \ + postgresql-client \ + curl \ + libvips \ + nodejs \ + \ + && npm install -g svgo \ + \ + # Install Passenger as a gem and compile the Apache module + \ + && gem install passenger --no-document \ + && yes | passenger-install-apache2-module --auto --languages ruby \ + && passenger-config validate-install --auto \ + \ + # Delete the build dependencies to reduce image size + \ + && apt-get purge -y --auto-remove $BUILD_DEPS \ + \ + # libgd-dev is requiered by the app on run time to process gps files + \ + && apt-get update && apt-get install -y --no-install-recommends libgd3 libgd-dev \ + \ + # Final cleanup + \ && apt-get clean && rm -rf /var/lib/apt/lists/* -COPY --from=builder /var/www /var/www -COPY --from=builder /usr/local/bundle /usr/local/bundle - -# Symlink tmp for Passenger -RUN ln -s /tmp /var/www/tmp # Apache configuration COPY config/production.conf /etc/apache2/sites-available/production.conf + +RUN passenger-install-apache2-module --snippet > /etc/apache2/mods-available/passenger.load && \ + passenger-config build-native-support + RUN a2enmod headers setenvif proxy proxy_http proxy_fcgi fcgid rewrite lbmethod_byrequests passenger && \ a2dissite 000-default && \ a2ensite production && \ echo "ServerName localhost" >> /etc/apache2/apache2.conf && \ apache2ctl configtest +RUN echo '#!/bin/bash\nexec /usr/local/bin/ruby --yjit --yjit-exec-mem-size=64 "$@"' > /usr/local/bin/ruby_yjit && \ + chmod +x /usr/local/bin/ruby_yjit + +WORKDIR $workdir + +COPY --chown=www-data:www-data --from=builder /var/www /$workdir +COPY --from=builder /usr/local/bundle /usr/local/bundle + COPY config/settings.yml $workdir/config/ COPY start.sh liveness.sh $workdir/ -RUN chmod +x $workdir/*.sh -RUN chown -R www-data:www-data /var/www + +RUN ln -s /tmp /var/www/tmp + +RUN mkdir -p /var/www/log && \ + touch /var/www/log/production.log && \ + chown -R www-data:www-data /var/www/log /var/www/public && \ + chown -R www-data:www-data /var/www + CMD ["./start.sh"] diff --git a/images/web/config/production.conf b/images/web/config/production.conf index e3d1f3da..7b6e0909 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -2,6 +2,7 @@ # ServerName localhost # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public + PassengerAppEnv production PassengerRuby /usr/local/bin/ruby RewriteEngine On @@ -12,22 +13,19 @@ RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Redirect to www openstreetmap.org - # RewriteCond %{HTTP_HOST} =openstreetmap.org - # RewriteCond %{HTTP_HOST} !^www\. [NC] - # RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + RewriteCond %{HTTP_HOST} =SERVER_DOMAIN_PLACEHOLDER + RewriteCond %{HTTP_HOST} !^www\. [NC] + RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] CGIPassAuth On SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 - # Proxying traffic to CGImap + #Proxying traffic to CGImap ProxyTimeout 1200 RewriteCond %{REQUEST_URI} ^/api/0\.6/map RewriteRule ^/api/0\.6/map(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] - - RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(node|way|relation|changeset)/[0-9]+(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/history(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ fcgi://${CGIMAP_URL}:${CGIMAP_PORT}$0 [P] @@ -53,4 +51,12 @@ FcgidIOTimeout 1200 FcgidConnectTimeout 1200 + + # Allow CORS for JSON, PBF, and PNG files for map-style + + Header set Access-Control-Allow-Origin "*" + Header set Access-Control-Allow-Methods "GET, OPTIONS" + Header set Access-Control-Allow-Headers "Content-Type" + + diff --git a/images/web/config/settings.yml b/images/web/config/settings.yml index 3e145e09..d6150b79 100644 --- a/images/web/config/settings.yml +++ b/images/web/config/settings.yml @@ -1,6 +1,6 @@ # The server protocol and host server_protocol: "http" -server_url: "openstreetmap.example.com" +server_url: "openstreetmap.example.com" # Publisher #publisher_url: "" # The generator @@ -8,6 +8,8 @@ generator: "OpenStreetMap server" copyright_owner: "OpenStreetMap and contributors" attribution_url: "http://www.openstreetmap.org/copyright" license_url: "http://opendatacommons.org/licenses/odbl/1-0/" +# Legal email address +legal_email: "legal@openstreetmap.org" # Support email address support_email: "openstreetmap@example.com" # Sender addresses for emails @@ -102,13 +104,13 @@ default_legale: GB # Location of data for attachments attachments_dir: ":rails_root/public/attachments" # Log file to use -#log_path: "" +log_path: "/var/www/log/production.log" # Log file to use for logstash #logstash_path: "" # List of memcache servers to use for caching memcache_servers: [] # URL of Nominatim instance to use for geocoding -nominatim_url: "https://nominatim-api.openstreetmap.org/" +nominatim_url: "https://nominatim.openstreetmap.org/" # Default editor default_editor: "id" # OAuth application for the web site @@ -131,6 +133,11 @@ overpass_credentials: false graphhopper_url: "https://graphhopper.com/api/1/route" fossgis_osrm_url: "https://routing.openstreetmap.de/" fossgis_valhalla_url: "https://valhalla1.openstreetmap.de/route" + +# Endpoints for Wikimedia integration +wikidata_api_url: "https://www.wikidata.org/w/api.php" +wikimedia_commons_url: "https://commons.wikimedia.org/wiki/" + # External authentication credentials #google_auth_id: "" #google_auth_secret: "" @@ -141,8 +148,15 @@ fossgis_valhalla_url: "https://valhalla1.openstreetmap.de/route" #github_auth_secret: "" #microsoft_auth_id: "" #microsoft_auth_secret: "" -#wikipedia_auth_id: "" -#wikipedia_auth_secret: "" +# wikipedia_auth_id: "" +# wikipedia_auth_secret: "" +#apple_auth_id: "" +#apple_team_id: "" +#apple_key_id: "" +#apple_private_key: "" +# openstreetmap_auth_id: "" +# openstreetmap_auth_secret: "" +# openstreetmap_auth_scopes: ["read_prefs"] # Thunderforest authentication details #thunderforest_key: "" # Tracestrack authentication details @@ -154,10 +168,10 @@ csp_enforce: false # URL for reporting Content-Security-Policy violations #csp_report_url: "" # Storage services to use in production mode -avatar_storage: "local" -trace_file_storage: "local" -trace_image_storage: "local" -trace_icon_storage: "local" +avatar_storage: "s3" # TODO: Change to S3 +trace_file_storage: "s3" # TODO: Change to S3 +trace_image_storage: "s3" # TODO: Change to S3 +trace_icon_storage: "s3" # TODO: Change to S3 # Root URL for storage services # avatar_storage_url: # trace_image_storage_url: @@ -185,3 +199,5 @@ doorkeeper_signing_key: | -----BEGIN PRIVATE KEY----- PRIVATE_KEY -----END PRIVATE KEY----- + +mastodon_url: "https://mapstodon.space/@osm" diff --git a/images/web/start.sh b/images/web/start.sh index a379fddb..83104976 100755 --- a/images/web/start.sh +++ b/images/web/start.sh @@ -30,12 +30,22 @@ EOF echo "S3 storage configuration set successfully." fi + #### Fix translation files: replace {مجتمع} with {community} to prevent KeyError + # This fixes the KeyError when template has Arabic placeholder but hash only has :community + find "$workdir/node_modules/osm-community-index/i18n" -name "*.yaml" -type f -exec sed -i 's/{مجتمع}/{community}/g' {} \; + + #### Initializing an empty $workdir/config/settings.local.yml file, typically used for development settings echo "" > $workdir/config/settings.local.yml #### Setting up server_url and server_protocol + SERVER_URL_CLEAN=$(echo "$SERVER_URL" | sed 's|/$||') sed -i -e 's/^server_protocol: ".*"/server_protocol: "'$SERVER_PROTOCOL'"/g' $workdir/config/settings.yml - sed -i -e 's/^server_url: ".*"/server_url: "'$SERVER_URL'"/g' $workdir/config/settings.yml + sed -i -e 's/^server_url: ".*"/server_url: "'$SERVER_URL_CLEAN'"/g' $workdir/config/settings.yml + + #### Extract domain from SERVER_URL and replace in production.conf + SERVER_DOMAIN=$(echo "$SERVER_URL_CLEAN" | sed -e 's|^[^/]*//||' -e 's|^www\.||' -e 's|/.*$||') + sed -i -e "s/SERVER_DOMAIN_PLACEHOLDER/$SERVER_DOMAIN/g" /etc/apache2/sites-available/production.conf ### Setting up website status sed -i -e 's/^status: ".*"/status: "'$WEBSITE_STATUS'"/g' $workdir/config/settings.yml @@ -76,6 +86,7 @@ EOF chmod 400 /var/www/private.pem export DOORKEEPER_SIGNING_KEY=$(cat /var/www/private.pem | sed -e '1d;$d' | tr -d '\n') sed -i "s#PRIVATE_KEY#${DOORKEEPER_SIGNING_KEY}#" $workdir/config/settings.yml + } restore_db() { @@ -99,6 +110,17 @@ start_background_jobs() { done } +log_and_tail() { + local file=$1 + if [ -f "$file" ]; then + echo "Logs from: $file" + tail -F "$file" & + else + echo "⚠️ Log file not found: $file" + fi +} + + setup_production() { setup_env_vars @@ -107,11 +129,8 @@ setup_production() { sleep 2 done - # echo "Running asset precompilation..." - # time bundle exec rake i18n:js:export assets:precompile - - echo "Copying static assets..." - cp "$workdir/public/leaflet-ohm-timeslider-v2/assets/"* "$workdir/public/assets/" + # Create the /passenger-instreg directory if it doesn’t exist. This is required in newer versions of Passenger. + mkdir -p /var/run/passenger-instreg echo "Running database migrations..." time bundle exec rails db:migrate @@ -121,12 +140,17 @@ setup_production() { ./cgimap.sh fi + echo "Logging and tailing logs..." + log_and_tail /var/www/log/production.log + log_and_tail /var/www/log/jobs_work.log + log_and_tail /var/log/apache2/error.log + log_and_tail /var/log/apache2/access.log + echo "Starting Apache server..." - apachectl -k start -DFOREGROUND & - start_background_jobs + start_background_jobs & + apachectl -k start -DFOREGROUND } - setup_development() { restore_db cp "$workdir/config/example.storage.yml" "$workdir/config/storage.yml"