stash

backend/backend/settings.py

@@ -158,6 +158,14 @@ STATIC_URL = '/static/'
 MEDIA_ROOT = os.environ.get('TOOLSHED_USERFILES_PATH', 'userfiles')
 MEDIA_URL = '/media/'
 
+# In prod, nginx (running as www-data) reads these files directly - see
+# SERVE_X_ACCEL_REDIRECT and playbook.yml's `location /redirect_media/`.
+# Pinned explicitly rather than left to the backend process's ambient umask,
+# so group-read (www-data is added to the backend's system group) is
+# guaranteed regardless of how the container is started.
+FILE_UPLOAD_PERMISSIONS = 0o640
+FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o750
+
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
 

deploy/prod/README.md

@@ -3,7 +3,10 @@
 `playbook.yml` automates installing docker.io and nginx (plus certbot, and
 obtaining/renewing a TLS certificate with it, on hosts that manage their own
 — see `behind_tls_proxy` below), building the backend, frontend and wiki
-images, exporting the frontend and wiki static builds for nginx to serve,
+images, exporting the frontend/wiki static builds and the backend's
+`collectstatic` output for nginx to serve directly (nginx also serves
+user-uploaded files directly, via an X-Accel-Redirect Django issues after its
+own permission check — see `location /redirect_media/` in `playbook.yml`),
 writing the small `/local/domains` and `/local/dns` fixture files the
 frontend fetches directly (registration domain list and DoH resolver
 preference — see `toolshed_register_domains`/`toolshed_doh_resolvers` in
@@ -51,8 +54,11 @@ toolshed:
   in `user@yourtoolshed.tld`) — see [DNS](#3-dns) for how those two relate.
 - `toolshed_handle_domain` — the **handle domain**, only needed when it's
   different from `toolshed_domain`. Omit it when the two are the same (it
-  then defaults to `toolshed_domain`). Set so nginx/Django accept requests
+  then defaults to `toolshed_domain`). It doesn't affect nginx/Django at all
-  for either domain, whichever ends up as the `Host` header.
+  (they only ever accept `toolshed_domain` as the `Host` header) — it's used
+  solely to populate the `/local/domains` registration fixture (see
+  `toolshed_register_domains` in `playbook.yml`); publishing the SRV record is a
+  separate, manual DNS step either way.
 - `toolshed_repo_url` — the git remote the playbook checks out and builds
   from. Required, no default.
 - `toolshed_version` — the branch, tag or commit to check out and build.
@@ -94,10 +100,12 @@ There are two distinct domains at play here, and it's easy to conflate them:
   `user@yourtoolshed.tld`. Toolshed usernames don't encode a server address
   directly; the frontend resolves the handle domain to a server via an SRV
   record, `_toolshed-server._tcp.<handle domain>.` (see
-  `frontend/src/store.js`, `lookupServer`). What's in `toolshed_handle_domain`
+  `frontend/src/store.js`, `lookupServer`), which always points at the web
-  (see [Per-deployment configuration](#2-per-deployment-configuration)) only
+  domain — nginx/Django never see the handle domain as a `Host` header.
-  makes nginx/Django accept it as a `Host` header — publishing the actual SRV
+  `toolshed_handle_domain` (see [Per-deployment
-  record is still a separate, manual DNS step, covered below.
+  configuration](#2-per-deployment-configuration)) only feeds the
+  `/local/domains` registration fixture; publishing the actual SRV record is
+  still a separate, manual DNS step, covered below.
 
 The SRV lookup happens for every login, not just federation with other
 servers, so **every** deployment needs it published for its own handle

deploy/prod/playbook.yml

@@ -41,6 +41,10 @@
     toolshed_backend_port: 8000
     toolshed_wiki_dist_dir: /var/www/toolshed-wiki
     toolshed_local_dir: /var/www/toolshed-local
+    # Django's collectstatic output (admin/drf-yasg assets etc.), exported
+    # from the built backend image so nginx can serve it directly instead of
+    # proxying to gunicorn for every asset request.
+    toolshed_static_dir: /var/www/toolshed-static
     # Domain(s) this server accepts registrations for (the "handle domain" -
     # see the README's DNS section). Served as a static /local/domains
     # fixture that the frontend's registration/pairing forms fetch to
@@ -73,15 +77,14 @@
     # header instead.
     toolshed_x_forwarded_proto: >-
       {{ '$http_x_forwarded_proto' if (behind_tls_proxy | default(false) | bool) else '$scheme' }}
-    # The web domain (toolshed_domain, mandatory) and the handle domain
+    # Only the web domain (toolshed_domain) - nginx server_name, Django
-    # (toolshed_handle_domain, optional - defaults to the web domain when
+    # ALLOWED_HOSTS, and the cert certbot requests. The handle domain
-    # they're the same) both need to be accepted by nginx/Django, since
+    # (toolshed_handle_domain) is resolved by clients via its own SRV record
-    # either may show up as the Host header depending on how the admin set
+    # and doesn't necessarily have an A record pointing at this host at all
-    # up DNS for this deployment. Deduplicated so setting them equal
+    # (see the README's DNS section), so it can't reliably serve an HTTP-01
-    # doesn't produce a repeated entry.
+    # challenge or ever show up as this nginx's Host header.
     toolshed_hostnames: >-
-      {{ [toolshed_domain | mandatory('toolshed_domain must be set as a host_var for ' ~ inventory_hostname),
+      {{ [toolshed_domain | mandatory('toolshed_domain must be set as a host_var for ' ~ inventory_hostname)] }}
-          toolshed_handle_domain | default(toolshed_domain)] | unique }}
     # Generated once per host on the controller and reused on every
     # subsequent run against that host, keyed by inventory_hostname so
     # separate deployments never end up sharing a Django SECRET_KEY.
@@ -120,6 +123,11 @@
               proxy_pass http://toolshed_backend;
           }
 
+          # Django (SignatureAuthentication + per-file friend/owner checks,
+          # see files/media_urls.py) decides whether the request is allowed
+          # at all; it never streams the bytes itself here (SERVE_X_ACCEL_REDIRECT
+          # is on), it just answers with an X-Accel-Redirect to the internal
+          # location below, which nginx follows and serves directly from disk.
           location /media {
               proxy_set_header Host $host;
               proxy_set_header X-Real-IP $remote_addr;
@@ -128,6 +136,19 @@
               proxy_pass http://toolshed_backend;
           }
 
+          # Only reachable via the X-Accel-Redirect above, never directly by
+          # clients (`internal`) - this is what makes it safe for nginx to
+          # serve these bytes itself without reimplementing the access
+          # checks Django already did in the /media location.
+          location /redirect_media/ {
+              internal;
+              alias {{ toolshed_data_dir }}/userfiles/;
+              # Django would normally set this itself (CORS_ALLOW_ALL_ORIGINS,
+              # see settings.py) but never gets to run for a request nginx
+              # serves directly - see the comment in files/media_urls.py.
+              add_header Access-Control-Allow-Origin * always;
+          }
+
           location /djangoadmin {
               proxy_set_header Host $host;
               proxy_set_header X-Real-IP $remote_addr;
@@ -144,8 +165,9 @@
               proxy_pass http://toolshed_backend;
           }
 
-          location /static {
+          location /static/ {
-              proxy_pass http://toolshed_backend/static;
+              alias {{ toolshed_static_dir }}/;
+              try_files $uri =404;
           }
 
           location /wiki/ {
@@ -283,6 +305,20 @@
         - "{{ toolshed_data_dir }}"
         - "{{ toolshed_data_dir }}/userfiles"
 
+    # nginx's `location /redirect_media/` (below) reads user-uploaded files
+    # straight off disk as www-data - group membership plus the 0750 mode
+    # above/FILE_UPLOAD_PERMISSIONS (backend/backend/settings.py) is what
+    # makes that readable without loosening it to world-readable.
+    - name: Allow nginx to read backend user files
+      ansible.builtin.user:
+        name: www-data
+        groups: toolshed
+        append: true
+      # New group membership only takes effect for processes started (or
+      # forked) after this - nginx's already-running workers won't see it
+      # until reloaded.
+      notify: reload nginx
+
     - name: Create frontend static output directory
       ansible.builtin.file:
         path: "{{ toolshed_dist_dir }}"
@@ -291,6 +327,14 @@
         group: www-data
         mode: "0755"
 
+    - name: Create backend static output directory
+      ansible.builtin.file:
+        path: "{{ toolshed_static_dir }}"
+        state: directory
+        owner: www-data
+        group: www-data
+        mode: "0755"
+
     - name: Write backend environment file
       ansible.builtin.copy:
         dest: "{{ toolshed_data_dir }}/backend.env"
@@ -306,7 +350,7 @@
           DEBUG={{ toolshed_debug }}
           SECRET_KEY={{ toolshed_secret_key }}
           ALLOWED_HOSTS={{ toolshed_hostnames | join(',') }}
-          SERVE_X_ACCEL_REDIRECT=False
+          SERVE_X_ACCEL_REDIRECT=True
           TOOLSHED_DB_PATH=/data/db.sqlite3
           TOOLSHED_USERFILES_PATH=/data/userfiles
       notify: restart backend
@@ -325,6 +369,26 @@
       changed_when: true
       notify: restart backend
 
+    # Dockerfile.backend runs collectstatic at build time, baking the result
+    # into the image at /app/staticfiles - copy it out to the host so nginx
+    # can serve it directly instead of proxying every asset request to
+    # gunicorn. No Django settings/DB access needed, so this can run as a
+    # one-off command against the image rather than the container.
+    - name: Export backend static files
+      ansible.builtin.command:
+        cmd: >-
+          docker run --rm -v {{ toolshed_static_dir }}:/output
+          {{ toolshed_backend_image }}:latest
+          sh -c "cp -a /app/staticfiles/. /output/"
+      changed_when: true
+
+    - name: Fix ownership of exported backend static files
+      ansible.builtin.file:
+        path: "{{ toolshed_static_dir }}"
+        owner: www-data
+        group: www-data
+        recurse: true
+
     - name: Install systemd unit for the backend container
       ansible.builtin.copy:
         dest: /etc/systemd/system/toolshed-backend.service

frontend/src/store.js

@@ -315,7 +315,12 @@ export default createStore({
             const request = '_toolshed-server._tcp.' + domain + '.'
             return await state.resolver.query(request, 'SRV').then(
                 (result) => result.map(
-                    (answer) => answer.target + ':' + answer.port))
+                    // Must match what the browser actually puts in the Host header for
+                    // the request this gets used to build (federation.js always signs
+                    // and fetches "https://" + server + target) - it omits a :443 for
+                    // the default HTTPS port, so keeping it here would make every
+                    // signature check on the receiving end fail against the real request.
+                    (answer) => answer.port === 443 ? answer.target : answer.target + ':' + answer.port))
         },
         async getHomeServers({state, dispatch, commit}) {
             if (state.home_servers)