Update nginx template to support horizontal scaling

This commit is part of bigger work to add horizontal scaling support to
PeerTube.

Author: Chocobozzz

support/nginx/peertube

@@ -1,14 +1,14 @@
 # Minimum Nginx version required:  1.13.0 (released Apr 25, 2017)
 # Please check your Nginx installation features the following modules via 'nginx -V':
-# STANDARD HTTP MODULES: Core, Proxy, Rewrite, Access, Gzip, Headers, HTTP/2, Log, Real IP, SSL, Thread Pool, Upstream, AIO Multithreading.
+# STANDARD HTTP MODULES: Core, Proxy, Rewrite, Access, Gzip, Headers, HTTP/2, Log, Real IP, SSL, Thread Pool, Upstream, AIO Multithreading, Map.
 # THIRD PARTY MODULES:   None.
 
 server {
   listen 80;
   listen [::]:80;
   server_name ${WEBSERVER_HOST};
 
-  location /.well-known/acme-challenge/ {
+  location ^~ /.well-known/acme-challenge/ {
     default_type "text/plain";
     root /var/www/certbot;
   }
@@ -19,29 +19,87 @@ upstream backend {
   server ${PEERTUBE_HOST};
 }
 
+##
+# ---------------------------------------------------------------------------
+# Horizontal scalability
+# ---------------------------------------------------------------------------
+#
+# Horizontal scalability allows to offload some API paths to `--role=secondary` processes.
+# See https://docs.joinpeertube.org/maintain/horizontal-scalability for more information.
+#
+# Leave this section untouched if you run a single PeerTube process.
+#
+# If you want to enable horizontal scalability, add one `server` line per secondary to both pools below
+# and keep `${PEERTUBE_HOST}` as a `backup` so reads survive every secondary going down.
+# For example:
+#
+#   upstream secondary_pool {
+#     server 127.0.0.1:9100;
+#     server 127.0.0.1:9101;
+#     server ${PEERTUBE_HOST} backup;
+#   }
+#
+# `secondary_views_pool` handles view/watch-time tracking only. It's optional since the heartbeat can be spread to any process,
+# but it's faster to stick a heartbeat to a single process (because of local process caching).
+# If a CDN or another load balancer sits in front, set `set_real_ip_from` / `real_ip_header` accordingly,
+# otherwise every viewer hashes to the same secondary and PeerTube counts every view against the proxy IP.
+##
+
+upstream secondary_pool {
+  server ${PEERTUBE_HOST};
+}
+
+upstream secondary_views_pool {
+  hash $remote_addr consistent;
+
+  server ${PEERTUBE_HOST};
+}
+
+# Secondary servers only support a subset of API endpoints
+# Leave this section untouched, even if you don't use horizontal scalability
+#
+# Regexes are evaluated top to bottom, first match wins
+# method + path -> pool
+map $request_method$uri $peertube_pool {
+  default                                                                       backend;
+
+  # Read-only video endpoints served by `--role=secondary` processes
+  "~^(GET|HEAD)/api/v1/videos/?$"                                               secondary_pool;
+  "~^(GET|HEAD)/api/v1/videos/(categories|licences|languages|privacies)$"       secondary_pool;
+  # Collections that live under /api/v1/videos/ but are not a video id: must be listed before the /api/v1/videos/<id> line below
+  "~^(GET|HEAD)/api/v1/videos/(blacklist|comments|imports|ownership|live)(/|$)" backend;
+  "~^(GET|HEAD)/api/v1/videos/[^/]+/?$"                                         secondary_pool;
+
+  # View/watch-time tracking: sticky to one process per viewer
+  "~^(PUT|POST)/api/v1/videos/[^/]+/(views|watching)$"                          secondary_views_pool;
+}
+
 server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
+
+  # On nginx >= 1.25.1 `http2` as a `listen` parameter is deprecated and logs a warning on startup
+  # If your nginx is recent enough, replace the two lines above by `listen 443 ssl;` + `listen [::]:443 ssl;`and add `http2 on;` here
   server_name ${WEBSERVER_HOST};
 
+  server_tokens off; # don't advertise the nginx version in responses and error pages
+
   access_log /var/log/nginx/peertube.access.log; # reduce I/0 with buffer=10m flush=5m
   error_log  /var/log/nginx/peertube.error.log;
 
   ##
   # Certificates
-  # you need a certificate to run in production. see https://letsencrypt.org/
   ##
   ssl_certificate     /etc/letsencrypt/live/${WEBSERVER_HOST}/fullchain.pem;
   ssl_certificate_key /etc/letsencrypt/live/${WEBSERVER_HOST}/privkey.pem;
 
-  location ^~ '/.well-known/acme-challenge' {
+  location ^~ /.well-known/acme-challenge/ {
     default_type "text/plain";
     root /var/www/certbot;
   }
 
   ##
-  # Security hardening (as of Nov 15, 2020)
-  # based on Mozilla Guideline v5.6
+  # Security hardening (as of Nov 15, 2020) based on Mozilla Guideline v5.6
   ##
 
   ssl_protocols             TLSv1.2 TLSv1.3;
@@ -55,20 +113,48 @@ server {
 
   ##
   # Application
+  # ---------------------------------------------------------------------------
+  #
+  # IMPORTANT, read before editing a `location` below.
+  #
+  # `try_files /dev/null @named` performs an internal redirect that adds subtleties:
+  #   * `client_max_body_size` and the `add_header` that are read from the location matching the URI, i.e., the `location ~ ...` block.
+  #   * `proxy_*` directives and `add_header` on a proxied response are read from the final configuration, i.e. from the `@named` block
   ##
 
   location @api {
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header Host            $host; # Use $http_host if you use a non standard HTTP port
     proxy_set_header X-Real-IP       $remote_addr;
 
-    client_max_body_size  100k; # default is 1M
+    proxy_connect_timeout 10m;
+    proxy_send_timeout    10m;
+    proxy_read_timeout    10m;
+    send_timeout          10m;
+
+    proxy_pass http://$peertube_pool;
+  }
+
+  # Same as @api, but streams the request body go directly to PeerTube instead of being written to /var/lib/nginx first
+  # Used by the upload endpoints, which are always served by the primary
+  location @api_unbuffered {
+    # Required for `proxy_request_buffering off`
+    proxy_http_version 1.1;
+
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host            $host; # Use $http_host if you use a non standard HTTP port
+    proxy_set_header X-Real-IP       $remote_addr;
 
     proxy_connect_timeout 10m;
     proxy_send_timeout    10m;
     proxy_read_timeout    10m;
     send_timeout          10m;
 
+    proxy_request_buffering off;
+    # Only applies to chunked request bodies
+    # A body with a Content-Length is checked against the `client_max_body_size` of the location that matched the URI, before we get here
+    client_max_body_size    0;
+
     proxy_pass http://backend;
   }
 
@@ -77,17 +163,15 @@ server {
   }
 
   location ~ ^/api/v1/videos/(upload-resumable|([^/]+/source/replace-resumable))$ {
-    client_max_body_size    0;
-    proxy_request_buffering off;
+    client_max_body_size 0;
 
-    try_files /dev/null @api;
+    try_files /dev/null @api_unbuffered;
   }
 
   location ~ ^/api/v1/users/[^/]+/imports/import-resumable$ {
-    client_max_body_size    0;
-    proxy_request_buffering off;
+    client_max_body_size 0;
 
-    try_files /dev/null @api;
+    try_files /dev/null @api_unbuffered;
   }
 
   location ~ ^/api/v1/videos/(upload|([^/]+/studio/edit))$ {
@@ -98,22 +182,28 @@ server {
     # This data gets stored in /var/lib/nginx by default, so you may want to put this directory
     # on a dedicated filesystem.
     client_max_body_size                      12G; # default is 1M
-    add_header            X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
-    proxy_request_buffering off;
+    add_header            X-File-Maximum-Size 8G always; # inform the client of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
 
-    try_files /dev/null @api;
+    try_files /dev/null @api_unbuffered;
   }
 
   location ~ ^/api/v1/runners/jobs/[^/]+/(update|success)$ {
-    client_max_body_size    0;
-    proxy_request_buffering off;
+    client_max_body_size 0;
+
+    try_files /dev/null @api_unbuffered;
+  }
+
+  # View tracking (PUT|POST /views and /watching)
+  location ~ ^/api/v1/videos/[^/]+/(views|watching)$ {
+    client_max_body_size 100k; # default is 1M
 
     try_files /dev/null @api;
   }
 
-  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) {
+  # Endpoints that accept an image (avatar, banner, thumbnail, preview, channel logo, instance logo)
+  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me|config/instance-(avatar|banner|logo)) {
     client_max_body_size                      12M; # default is 1M
-    add_header            X-File-Maximum-Size 8M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
+    add_header            X-File-Maximum-Size 8M always; # inform the client of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
 
     try_files /dev/null @api;
   }
@@ -152,14 +242,13 @@ server {
 
   ##
   # Performance optimizations
-  # For extra performance please refer to https://github.com/denji/nginx-tuning
   ##
 
   root /var/www/peertube/storage;
 
-  # Enable compression for JS/CSS/HTML, for improved client load times.
-  # It might be nice to compress JSON/XML as returned by the API, but
-  # leaving that out to protect against potential BREACH attack.
+  # Compress the static client assets and the public XML feeds, for improved client load times.
+  #
+  # `application/json` is deliberately left out to prevent BREACH (CVE-2013-3587)
   gzip              on;
   gzip_vary         on;
   gzip_types        # text/html is always compressed by HttpGzipModule
@@ -169,7 +258,11 @@ server {
                     font/opentype
                     application/vnd.ms-fontobject
                     image/svg+xml
-                    application/xml;
+                    # PeerTube negotiates any of these for /feeds/* and /sitemap.xml
+                    text/xml
+                    application/xml
+                    application/rss+xml
+                    application/atom+xml;
   gzip_min_length   1000; # default is 20 bytes
   gzip_buffers      16 8k;
   gzip_comp_level   2; # default is 1
@@ -231,22 +324,18 @@ server {
     # Or this line with nginx < 1.17.0
     # set $limit_rate $peertube_limit_rate;
 
+    # These files are public and unauthenticated, so they can be read cross origin.
+    add_header Access-Control-Allow-Origin  '*' always;
+    add_header Access-Control-Allow-Methods 'GET, OPTIONS' always;
+    add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' always;
+    add_header Access-Control-Max-Age       1728000 always; # Preflight request can be cached 20 days
+
+    # Keep this `if` free of `add_header`: a taken `if` block replaces the whole add_header set of
+    # the enclosing location, which would strip the CORS headers above from preflight responses.
     if ($request_method = 'OPTIONS') {
-      add_header Access-Control-Allow-Origin  '*';
-      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
-      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
-      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
-      add_header Content-Type                 'text/plain; charset=UTF-8';
-      add_header Content-Length               0;
       return 204;
     }
 
-    if ($request_method = 'GET') {
-      add_header Access-Control-Allow-Origin  '*';
-      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
-      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
-    }
-
     # Enabling the sendfile directive eliminates the step of copying the data into the buffer
     # and enables direct copying data from one file descriptor to another.
     sendfile on;