j3d1/toolshed
2
1
Fork 0
Code Issues 9 Pull requests 1 Projects Releases Packages Wiki Activity

Snapshot: alpha-2026-9

Browse source
This commit is contained in:
j3d1 2026-09-02 21:40:28 +02:00
parent 9acf5a97e2
commit d00b5c7961
241 changed files with 85546 additions and 2409 deletions
Download patch file Download diff file Expand all files Collapse all files

3
deploy/prod/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
.secrets/
inventory.yml
.frontend-build/

33
deploy/prod/Dockerfile.backend Normal file
View file

@ -0,0 +1,33 @@
# Production image for the Django backend.
# Runs migrations then serves the app with gunicorn on port 8000.
# Static files are collected at build time into /app/staticfiles and
# served by the backend itself behind the host nginx reverse proxy.
FROM python:3.11-slim
# The build context here is just backend/ (no .git), so settings.py's own
# `git rev-parse` fallback can't find a repo - the actual commit is passed
# in from the real checkout via this build-arg instead (see playbook.yml).
ARG GIT_COMMIT=unknown
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
DJANGO_SETTINGS_MODULE=backend.settings \
GIT_COMMIT=$GIT_COMMIT
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r requirements.txt gunicorn
COPY . .
# collectstatic only needs Django settings to import cleanly, not a real
# secret; the actual SECRET_KEY is injected at container runtime via
# --env-file and overrides this.
RUN SECRET_KEY=build-time-placeholder python manage.py collectstatic --noinput
EXPOSE 8000
CMD ["sh", "-c", "python manage.py migrate --noinput && exec gunicorn backend.wsgi:application --bind 0.0.0.0:8000 --workers 3"]

26
deploy/prod/Dockerfile.frontend Normal file
View file

@ -0,0 +1,26 @@
# Build-only image for the Vue frontend.
# It is never run as a service: ansible builds this image once, runs it
# with the host output directory bind-mounted at /output, the container
# copies the compiled static build into it, and exits. Nginx on the host
# then serves that directory directly.
FROM node:20-alpine AS build
WORKDIR /app
# The build context here is just frontend/ (no .git), so vite.config.js's
# own `git rev-parse` fallback can't find a repo - the actual commit is
# passed in from the real checkout via this build-arg instead (see
# playbook.yml).
ARG GIT_COMMIT=unknown
ENV GIT_COMMIT=$GIT_COMMIT
COPY package.json package-lock.json ./
#COPY extras/ ./extras/
RUN npm ci
COPY . .
RUN npm run build
FROM alpine AS export
COPY --from=build /app/dist /dist
VOLUME /output
CMD ["sh", "-c", "rm -rf /output/* && cp -a /dist/. /output/"]

17
deploy/prod/Dockerfile.wiki Normal file
View file

@ -0,0 +1,17 @@
# Build-only image for the project wiki (mkdocs).
# It is never run as a service: ansible builds this image once, runs it
# with the host output directory bind-mounted at /output, the container
# copies the built static site into it, and exits. Nginx on the host
# then serves that directory directly, the same way it does the frontend.
FROM python:3.11-slim AS build
WORKDIR /wiki
RUN pip install --no-cache-dir mkdocs
COPY mkdocs.yml ./
COPY docs/ ./docs/
RUN mkdocs build
FROM alpine AS export
COPY --from=build /wiki/site /site
VOLUME /output
CMD ["sh", "-c", "rm -rf /output/* && cp -a /site/. /output/"]

224
deploy/prod/README.md Normal file
View file

@ -0,0 +1,224 @@
# Toolshed production deployment — manual steps
`playbook.yml` automates installing docker.io and nginx (plus certbot, and
obtaining/renewing a TLS certificate with it, on hosts that manage their own
— see `behind_tls_proxy` below), building the backend, frontend and wiki
images, exporting the frontend/wiki static builds and the backend's
`collectstatic` output for nginx to serve directly (nginx also serves
user-uploaded files directly, via an X-Accel-Redirect Django issues after its
own permission check — see `location /redirect_media/` in `playbook.yml`),
writing the small `/local/domains` and `/local/dns` fixture files the
frontend fetches directly (registration domain list and DoH resolver
preference — see `toolshed_register_domains`/`toolshed_doh_resolvers` in
`playbook.yml`), configuring nginx, and installing the `toolshed-backend`
systemd service. It does **not** set up the target server or DNS. Those are
manual, one-time steps and are covered here. Seeding the backend's shared
reference data is also a manual, one-time step — see
[First superuser & shared reference data](#5-first-superuser--shared-reference-data).
## 1. Server & firewall
- A Debian/Ubuntu host reachable over SSH.
- Copy `inventory.example.yml` to `inventory.yml` (git-ignored, since it
holds real hostnames/IPs) and fill in your host(s) — see
[Per-deployment configuration](#2-per-deployment-configuration).
- Inbound TCP 80 open in the firewall/security group. Also open 443 unless
`behind_tls_proxy: true` — and keep both open permanently, not just for the
initial deploy: certbot's renewal timer needs 80 for the ACME HTTP-01
challenge and 443 for HTTPS traffic for as long as this host is live.
## 2. Per-deployment configuration
Each entry under `hosts:` in `inventory.yml` is its own independent
deployment (its own repo checkout, database, domain, systemd service and
Django `SECRET_KEY` — nothing is shared between hosts). Set these as
host_vars directly on each host entry, not via `-e` on the command line,
so a single `inventory.yml` can hold several unrelated deployments safely:
```yaml
toolshed:
hosts:
my-server:
ansible_host: 203.0.113.10
ansible_user: deploy
toolshed_domain: toolshed.webdomain.tld
toolshed_handle_domain: yourtoolshed.tld # optional, see below
toolshed_repo_url: git@example.com:your-org/toolshed.git
behind_tls_proxy: false
```
- `toolshed_domain` — the **web domain**: the nginx `server_name`, Django
`ALLOWED_HOSTS`, and the hostname(s) you'll point a TLS cert at — e.g.
`toolshed.webdomain.tld`. Required, no default. May be a single domain (as
above) or a list, e.g. to also answer on a `www.` alias:
```yaml
toolshed_domain:
- toolshed.webdomain.tld
- www.toolshed.webdomain.tld
```
The Let's Encrypt certificate covers all of them, named on disk after
whichever one is listed first. This is not necessarily the same as the
**handle domain** your users log in with (the part after `@` in
`user@yourtoolshed.tld`) — see [DNS](#3-dns) for how those two relate.
- `toolshed_handle_domain` — the **handle domain**, only needed when it's
different from `toolshed_domain`. Omit it when the two are the same (it
then defaults to `toolshed_domain`). Like `toolshed_domain`, it may be a
single domain or a list, e.g. if this deployment accepts registrations for
more than one handle domain. It doesn't affect nginx/Django at all (they
only ever accept `toolshed_domain` as the `Host` header) — it's used
solely to populate the `/local/domains` registration fixture (see
`toolshed_register_domains` in `playbook.yml`); publishing the SRV record for
each handle domain is a separate, manual DNS step either way.
- `toolshed_repo_url` — the git remote the playbook checks out and builds
from. Required, no default.
- `toolshed_version` — the branch, tag or commit to check out and build.
Optional, defaults to `stable`.
- `behind_tls_proxy` — `true` if TLS for this host is already terminated by
something in front of it (e.g. an external reverse proxy or load
balancer) that forwards plain HTTP here; `false` if this nginx has to
terminate TLS itself. This controls two things:
- Whether nginx trusts an upstream `X-Forwarded-Proto` header or sets its
own — get this wrong and Django's `SECURE_PROXY_SSL_HEADER` check
(`backend/backend/settings.py`) will treat every request as insecure or,
flipped the other way, treat plain HTTP as secure.
- Whether the playbook manages TLS at all. When `false`, it automatically
obtains a Let's Encrypt certificate via certbot and switches nginx over
to it — nothing to do manually beyond DNS (below). certbot's own systemd
timer keeps renewing it afterwards, independent of the playbook.
- `toolshed_letsencrypt_email` — required whenever `behind_tls_proxy` is
`false`; the account email certbot registers the certificate under
(used only for renewal-failure notices). Ignored otherwise.
- `http_port` — optional, defaults to `80`. Only relevant when
`behind_tls_proxy: true` and whatever's in front of this host forwards to
a nonstandard port instead of 80.
- `doh_resolvers` — optional, defaults to `["1.1.1.1", "8.8.8.8"]` (the same
hardcoded fallback the frontend itself uses, see `frontend/src/dns.js`).
DNS-over-HTTPS resolvers the frontend uses to look up a handle domain's
`_toolshed-server._tcp` SRV record before it has a cached preference.
Written to `/local/dns` at deploy time; only worth overriding as a
host_var (or `-e doh_resolvers='["9.9.9.9"]'`) if you want this
deployment to prefer a specific resolver.
## 3. DNS
There are two distinct domains at play here, and it's easy to conflate them:
- **Web domain** — the machine's actual hostname: nginx `server_name`,
Django `ALLOWED_HOSTS`, your TLS cert, what's in `toolshed_domain`. This is
what an A/AAAA record has to resolve to the server's IP for.
- **Handle domain** — the part after the `@` in a username, e.g.
`user@yourtoolshed.tld`. Toolshed usernames don't encode a server address
directly; the frontend resolves the handle domain to a server via an SRV
record, `_toolshed-server._tcp.<handle domain>.` (see
`frontend/src/store.js`, `lookupServer`), which always points at the web
domain — nginx/Django never see the handle domain as a `Host` header.
`toolshed_handle_domain` (see [Per-deployment
configuration](#2-per-deployment-configuration)) only feeds the
`/local/domains` registration fixture; publishing the actual SRV record is
still a separate, manual DNS step, covered below.
The SRV lookup happens for every login, not just federation with other
servers, so **every** deployment needs it published for its own handle
domain — even a standalone server that only ever serves itself.
These two domains can be **the same** or **completely different**, and
that's exactly the choice between an A record and an SRV record:
- **Same domain**: if `yourtoolshed.tld` is both the web domain and the
handle domain, it needs both an A record (so the domain itself resolves to
the server) and an SRV record that happens to point back at itself.
- **Different domains**: the handle domain only needs the SRV record — no A
record of its own — pointing at whatever web domain the server actually
lives at. This is useful when the handle you give out (short, brandable,
independent of hosting) shouldn't have to match wherever the box is
actually deployed (a subdomain of a shared hosting provider, an internal
service name, etc.).
**a) A/AAAA record — web domain → server IP:**
```sh
dig <your-web-domain> A
```
**b) SRV record — handle domain → web domain + port.** Use port 443: the
federation protocol is HTTPS-only.
```sh
dig _toolshed-server._tcp.<your-handle-domain> SRV
```
For example, with a handle domain of `yourtoolshed.tld` and a web domain of
`toolshed.webdomain.tld`:
```
$ dig _toolshed-server._tcp.yourtoolshed.tld srv
_toolshed-server._tcp.yourtoolshed.tld. 300 IN SRV 10 10 443 toolshed.webdomain.tld.
$ dig toolshed.webdomain.tld A
toolshed.webdomain.tld. 300 IN A 203.0.113.10
```
If you instead want `yourtoolshed.tld` itself to be the web domain too, its
SRV record just points at itself (`... SRV 10 10 443 yourtoolshed.tld.`) and
it additionally needs its own A record.
## 4. Secrets
`toolshed_secret_key` is generated once per host by the playbook (via the
`password` lookup, keyed by the host's inventory name) and stored as
`.secrets/<inventory-hostname>_secret_key` on the *control* machine, not on
the target. Back these files up — losing one invalidates all sessions and
signed cookies for that deployment on its next redeploy. They're git-ignored
on purpose; never commit them.
## 5. First superuser & shared reference data
The production backend image only runs `migrate` and `collectstatic` at
startup (see `Dockerfile.backend`) — unlike the dev compose setup, it never
runs the interactive `configure.py`. Two things dev gets "for free" from that
script therefore need doing manually, once, after a host's backend container
is first up (run these on the target host itself, or prefix with
`ssh <that-host>`):
- **Superuser account:**
```sh
docker exec -it toolshed-backend python manage.py createsuperuser
```
- **Shared reference data** (the standard categories/properties/tags
shipped in `backend/shared_data/*.json` — tools, electrical, screws, IT,
etc.): without this step a fresh deployment starts with none of them.
Run `configure.py` interactively (the `-it` flags matter — the script's
prompts only appear with a real tty) and answer "yes" when it asks to
import them:
```sh
docker exec -it toolshed-backend python configure.py
```
The other prompts it asks first (create `.env`, create a database) are
harmless to answer "yes" to as well: the container already gets its real
`SECRET_KEY`/`ALLOWED_HOSTS`/db path from the environment (the systemd unit
passes them via `--env-file`, see the "Write backend environment file" task
in `playbook.yml`), those checks just look for files at paths relative to
`/app` that don't exist in this container, and re-running `migrate` against
the real database is idempotent. You can say "no" to the superuser prompt
here if you already created one above.
## 6. Running the playbook
Always target one host at a time with `--limit` — running against the whole
`toolshed` group in one invocation would apply every host's own
`toolshed_domain`/`toolshed_repo_url` correctly (they're per-host vars, see
[Per-deployment configuration](#2-per-deployment-configuration)), but rolls
out all deployments back-to-back in one run, which is rarely what you want:
```sh
ansible-playbook -i inventory.yml playbook.yml --limit my-server
```
Re-run it to roll out a new version to that host. It deploys whatever
`toolshed_version` is set for that host (`stable` by default) — set the
host_var for a persistent change, or pass `-e toolshed_version=<branch/tag/commit>`
for a one-off deploy of something else.

53
deploy/prod/inventory.example.yml Normal file
View file

@ -0,0 +1,53 @@
---
# Copy this file to inventory.yml (git-ignored) and fill in your real
# hosts. Each entry under hosts: is an independent deployment - see the
# README's "Per-deployment configuration" section for what each var means.
toolshed:
hosts:
my-server:
ansible_host: 203.0.113.10
ansible_user: deploy
# toolshed_domain is the "web domain" - see the README's DNS section
# for how this relates to the separate "handle domain" your users
# log in with (user@yourtoolshed.tld). May be a single domain (as
# here) or a list, e.g. to also answer on a "www." alias:
# toolshed_domain:
# - toolshed.webdomain.tld
# - www.toolshed.webdomain.tld
# The Let's Encrypt certificate is requested for all of them, named
# after whichever one is listed first.
toolshed_domain: toolshed.webdomain.tld
# Optional - only needed if the handle domain differs from the web
# domain above. Omit it entirely when they're the same. Like
# toolshed_domain, this may be a single domain or a list, e.g. if this
# deployment accepts registrations for more than one handle domain:
# toolshed_handle_domain:
# - yourtoolshed.tld
# - alt.yourtoolshed.tld
toolshed_handle_domain: yourtoolshed.tld
toolshed_repo_url: git@example.com:your-org/toolshed.git
# Optional - branch, tag or commit to deploy. Defaults to "stable".
toolshed_version: stable
# true if something in front of this host already terminates TLS
# (reverse proxy/load balancer), false if this nginx must do it itself.
behind_tls_proxy: false
# Required whenever behind_tls_proxy is false: the playbook obtains
# its own Let's Encrypt certificate via certbot, which needs an
# account email for renewal notices.
toolshed_letsencrypt_email: admin@example.com
# A second, unrelated deployment behind an existing TLS-terminating
# proxy - remove this if you only run one instance. Here the handle
# domain and web domain are the same, so toolshed_handle_domain is
# simply omitted, and toolshed_letsencrypt_email isn't needed since
# this nginx never handles TLS itself.
my-other-server:
ansible_host: my-other-server.example.com
ansible_user: deploy
toolshed_domain: toolshed.example.com
toolshed_repo_url: git@example.com:your-org/toolshed.git
behind_tls_proxy: true
# Only needed if the proxy in front forwards to something other than
# port 80 on this host.
http_port: 8080

677
deploy/prod/playbook.yml Normal file
View file

@ -0,0 +1,677 @@
---
# Production deploy for toolshed.
#
# - installs docker.io and nginx on the target (plus certbot, unless
# behind_tls_proxy is true)
# - checks out the source and builds the backend and frontend docker images
# - runs the frontend image once to export its static build, which nginx
# then serves directly (the frontend image is never run as a service)
# - configures nginx (inline template, no separate .conf file) and, unless
# behind_tls_proxy is true, obtains/renews a Let's Encrypt certificate via
# certbot and switches nginx over to it automatically - no manual TLS step
# - installs and manages a systemd service that runs the backend container
#
# Usage (each host is its own independent deployment - always target one
# at a time, never the whole "toolshed" group in one run):
# ansible-playbook -i inventory.yml playbook.yml --limit my-server
#
# toolshed_repo_url, toolshed_domain, toolshed_handle_domain (optional,
# either may be a single domain or a list of domains), toolshed_version
# (optional, defaults to "stable"), behind_tls_proxy and
# toolshed_letsencrypt_email (required unless behind_tls_proxy is true) are
# per-deployment and must be set as host_vars in inventory.yml (copy
# inventory.example.yml) rather than here or via -e, so that each host in
# the "toolshed" group can point at its own repo/domain/branch. They're read
# with `mandatory`/`default()` below instead of being declared in play
# `vars:`, since play vars always take precedence over inventory host_vars
# and would otherwise silently override whatever is set per-host.
- name: Deploy toolshed
hosts: toolshed
become: true
vars:
toolshed_src_dir: /opt/toolshed/src
toolshed_data_dir: /opt/toolshed/data
toolshed_dist_dir: /var/www/toolshed
toolshed_backend_image: toolshed-backend
toolshed_frontend_image: toolshed-frontend-builder
toolshed_wiki_image: toolshed-wiki-builder
toolshed_backend_container: toolshed-backend
toolshed_backend_port: 8000
toolshed_wiki_dist_dir: /var/www/toolshed-wiki
toolshed_local_dir: /var/www/toolshed-local
# The frontend build runs on the controller (see "Build frontend builder
# docker image (controller)" below) rather than the target host, so its
# scratch checkout and build output live here instead of under
# toolshed_src_dir/toolshed_dist_dir. Keyed by inventory_hostname so
# concurrent deploys to different hosts never collide.
toolshed_frontend_build_src_dir: "{{ playbook_dir }}/.frontend-build/{{ inventory_hostname }}/src"
toolshed_frontend_build_dist_dir: "{{ playbook_dir }}/.frontend-build/{{ inventory_hostname }}/dist"
# Django's collectstatic output (admin/drf-yasg assets etc.), exported
# from the built backend image so nginx can serve it directly instead of
# proxying to gunicorn for every asset request.
toolshed_static_dir: /var/www/toolshed-static
# Domain(s) this server accepts registrations for (the "handle domain" -
# see the README's DNS section). toolshed_handle_domain may be a single
# domain or a list; when unset it falls back to toolshed_domain (whole
# list, if that's a list too). Served as a static /local/domains fixture
# that the frontend's registration/pairing forms fetch to populate their
# domain dropdown (frontend/src/views/Register.vue, Pairing.vue) -
# without it that dropdown is just empty.
toolshed_handle_domain_or_default: "{{ toolshed_handle_domain | default(toolshed_domain) }}"
toolshed_register_domains: >-
{{ ([toolshed_handle_domain_or_default]
if toolshed_handle_domain_or_default is string
else toolshed_handle_domain_or_default) | unique }}
# DoH resolvers the frontend falls back to for SRV lookups when it has
# no cached preference yet, served as a static /local/dns fixture. These
# match the frontend's own hardcoded fallback (frontend/src/dns.js), so
# this mostly makes the choice explicit and per-host overridable (e.g.
# -e doh_resolvers='["9.9.9.9"]') rather than changing behavior.
toolshed_doh_resolvers: "{{ doh_resolvers | default(['1.1.1.1', '8.8.8.8']) }}"
# Docker tags can't contain "/", but toolshed_version is a git ref and
# branch names like "jedi/proto/frontend" do - sanitize before using it
# as an image tag. The raw value is still used as-is for the actual git
# checkout, where slashes are fine.
toolshed_image_tag: "{{ (toolshed_version | default('stable')) | replace('/', '-') }}"
toolshed_debug: "False"
# Plain HTTP listen port. Only relevant behind an external proxy that
# forwards to something other than 80 (see http_port in inventory.yml);
# when this nginx terminates TLS itself, the public port is always 443.
toolshed_http_port: "{{ http_port | default(80) }}"
toolshed_letsencrypt_webroot: /var/www/letsencrypt
# Nginx sets its own X-Forwarded-Proto from $scheme when it terminates
# TLS itself. Behind an external TLS-terminating proxy, $scheme at this
# nginx is always "http" (the proxy already stripped TLS one hop
# earlier), so overwriting the header with $scheme would tell Django
# every request is insecure. In that case pass through the proxy's own
# header instead.
toolshed_x_forwarded_proto: >-
{{ '$http_x_forwarded_proto' if (behind_tls_proxy | default(false) | bool) else '$scheme' }}
# Only the web domain (toolshed_domain) - nginx server_name, Django
# ALLOWED_HOSTS, and the cert certbot requests. The handle domain
# (toolshed_handle_domain) is resolved by clients via its own SRV record
# and doesn't necessarily have an A record pointing at this host at all
# (see the README's DNS section), so it can't reliably serve an HTTP-01
# challenge or ever show up as this nginx's Host header.
#
# toolshed_domain may be a single domain or a list (e.g. a bare domain
# plus a "www." alias). certbot names the Let's Encrypt certificate's
# live/ directory after whichever domain is passed first via -d, so
# toolshed_hostnames[0] (below) is used wherever the playbook needs to
# reference that directory by name.
toolshed_domain_checked: >-
{{ toolshed_domain | mandatory('toolshed_domain must be set as a host_var for ' ~ inventory_hostname) }}
toolshed_hostnames: >-
{{ ([toolshed_domain_checked]
if toolshed_domain_checked is string
else toolshed_domain_checked) | unique }}
# Generated once per host on the controller and reused on every
# subsequent run against that host, keyed by inventory_hostname so
# separate deployments never end up sharing a Django SECRET_KEY.
toolshed_secret_key: >-
{{ lookup('ansible.builtin.password',
playbook_dir ~ '/.secrets/' ~ inventory_hostname ~ '_secret_key length=64 chars=ascii_letters,digits') }}
# Rendered twice against the same var (see the tasks below): once before
# a certificate exists (serves the site plainly over toolshed_http_port,
# or over 80/plain-HTTP forever if behind_tls_proxy), and once after
# certbot has obtained one, at which point the plain HTTP vhost switches
# to a redirect and a 443 vhost with the real content appears. Whichever
# of those two states applies, toolshed_cert (a registered `stat` result,
# undefined/false until it's checked) decides which one renders - this
# is the "another nginx config" from a single inline template, driven by
# behind_tls_proxy and certificate state rather than a separate file.
toolshed_nginx_conf: |
upstream toolshed_backend {
server 127.0.0.1:{{ toolshed_backend_port }};
}
{% macro toolshed_locations() %}
location /api {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /auth {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
# Django (SignatureAuthentication + per-file friend/owner checks,
# see files/media_urls.py) decides whether the request is allowed
# at all; it never streams the bytes itself here (SERVE_X_ACCEL_REDIRECT
# is on), it just answers with an X-Accel-Redirect to the internal
# location below, which nginx follows and serves directly from disk.
location /media {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
# Only reachable via the X-Accel-Redirect above, never directly by
# clients (`internal`) - this is what makes it safe for nginx to
# serve these bytes itself without reimplementing the access
# checks Django already did in the /media location.
location /redirect_media/ {
internal;
alias {{ toolshed_data_dir }}/userfiles/;
# Django would normally set this itself (CORS_ALLOW_ALL_ORIGINS,
# see settings.py) but never gets to run for a request nginx
# serves directly - see the comment in files/media_urls.py.
add_header Access-Control-Allow-Origin * always;
}
location /djangoadmin {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /docs {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /static/ {
alias {{ toolshed_static_dir }}/;
try_files $uri =404;
}
location /wiki/ {
alias {{ toolshed_wiki_dist_dir }}/;
try_files $uri $uri/ =404;
}
location = /wiki {
return 301 /wiki/;
}
# Static fixtures the frontend fetches directly (registration
# domain list, DoH resolver preference) - see toolshed_register_domains
# and toolshed_doh_resolvers above.
location /local/ {
alias {{ toolshed_local_dir }}/;
try_files $uri.json =404;
add_header Content-Type application/json;
}
# Vue-router history mode: fall back to index.html for
# any path that isn't a real static file.
location / {
try_files $uri $uri/ /index.html;
}
{% endmacro %}
{% if behind_tls_proxy | default(false) | bool %}
server {
listen {{ toolshed_http_port }};
listen [::]:{{ toolshed_http_port }};
server_name {{ toolshed_hostnames | join(' ') }};
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
}
{% else %}
{% set tls_active = toolshed_cert.stat.exists | default(false) %}
server {
listen {{ toolshed_http_port }};
listen [::]:{{ toolshed_http_port }};
server_name {{ toolshed_hostnames | join(' ') }};
location /.well-known/acme-challenge/ {
root {{ toolshed_letsencrypt_webroot }};
}
{% if tls_active %}
location / {
return 301 https://$host$request_uri;
}
{% else %}
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
{% endif %}
}
{% if tls_active %}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name {{ toolshed_hostnames | join(' ') }};
ssl_certificate /etc/letsencrypt/live/{{ toolshed_hostnames[0] }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ toolshed_hostnames[0] }}/privkey.pem;
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
}
{% endif %}
{% endif %}
tasks:
- name: Install docker.io, nginx and rsync
ansible.builtin.apt:
name:
- docker.io
- nginx
# rsync is what the frontend dist sync (further down) relies on -
# it's the ansible.posix.synchronize module's transport.
- rsync
state: present
update_cache: true
- name: Install certbot
ansible.builtin.apt:
name: certbot
state: present
when: not (behind_tls_proxy | default(false) | bool)
- name: Ensure docker is running and enabled
ansible.builtin.systemd:
name: docker
state: started
enabled: true
- name: Ensure nginx is running and enabled
ansible.builtin.systemd:
name: nginx
state: started
enabled: true
- name: Checkout toolshed source
ansible.builtin.git:
repo: "{{ toolshed_repo_url | mandatory('toolshed_repo_url must be set as a host_var for ' ~ inventory_hostname) }}"
dest: "{{ toolshed_src_dir }}"
version: "{{ toolshed_version | default('stable') }}"
force: true
# frontend/extras is registered as a submodule but unused and its
# pinned commit isn't fetchable from upstream - don't let a broken
# submodule block the checkout.
recursive: false
register: toolshed_checkout
- name: Create toolshed system user
ansible.builtin.user:
name: toolshed
system: true
shell: /usr/sbin/nologin
home: "{{ toolshed_data_dir }}"
create_home: false
register: toolshed_user
- name: Create backend data directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: toolshed
group: toolshed
mode: "0750"
loop:
- "{{ toolshed_data_dir }}"
- "{{ toolshed_data_dir }}/userfiles"
# nginx's `location /redirect_media/` (below) reads user-uploaded files
# straight off disk as www-data - group membership plus the 0750 mode
# above/FILE_UPLOAD_PERMISSIONS (backend/backend/settings.py) is what
# makes that readable without loosening it to world-readable.
- name: Allow nginx to read backend user files
ansible.builtin.user:
name: www-data
groups: toolshed
append: true
# New group membership only takes effect for processes started (or
# forked) after this - nginx's already-running workers won't see it
# until reloaded.
notify: reload nginx
- name: Create frontend static output directory
ansible.builtin.file:
path: "{{ toolshed_dist_dir }}"
state: directory
owner: "www-data"
group: "www-data"
mode: "0750"
- name: Create backend static output directory
ansible.builtin.file:
path: "{{ toolshed_static_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0750"
- name: Write backend environment file
ansible.builtin.copy:
dest: "{{ toolshed_data_dir }}/backend.env"
# Root-owned and unreadable by the toolshed user on purpose: this is
# read by the docker daemon (root) via --env-file at container
# start and injected directly as env vars, so the containerized app
# - which runs as the toolshed user, see the systemd unit below -
# never needs filesystem access to its own SECRET_KEY.
owner: root
group: root
mode: "0600"
content: |
DEBUG={{ toolshed_debug }}
SECRET_KEY={{ toolshed_secret_key }}
ALLOWED_HOSTS={{ toolshed_hostnames | join(',') }}
SERVE_X_ACCEL_REDIRECT=True
TOOLSHED_DB_PATH=/data/db.sqlite3
TOOLSHED_USERFILES_PATH=/data/userfiles
notify: restart backend
- name: Build backend docker image
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_backend_image }}:{{ toolshed_image_tag }}
--build-arg GIT_COMMIT={{ toolshed_checkout.after[:7] }}
-f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.backend {{ toolshed_src_dir }}/backend
changed_when: true
notify: restart backend
- name: Tag backend image as latest
ansible.builtin.command:
cmd: docker tag {{ toolshed_backend_image }}:{{ toolshed_image_tag }} {{ toolshed_backend_image }}:latest
changed_when: true
notify: restart backend
# Dockerfile.backend runs collectstatic at build time, baking the result
# into the image at /app/staticfiles - copy it out to the host so nginx
# can serve it directly instead of proxying every asset request to
# gunicorn. No Django settings/DB access needed, so this can run as a
# one-off command against the image rather than the container.
- name: Export backend static files
ansible.builtin.command:
cmd: >-
docker run --rm -v {{ toolshed_static_dir }}:/output
{{ toolshed_backend_image }}:latest
sh -c "cp -a /app/staticfiles/. /output/"
changed_when: true
- name: Fix ownership of exported backend static files
ansible.builtin.file:
path: "{{ toolshed_static_dir }}"
owner: www-data
group: www-data
recurse: true
- name: Install systemd unit for the backend container
ansible.builtin.copy:
dest: /etc/systemd/system/toolshed-backend.service
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=Toolshed backend (Django) container
After=docker.service network-online.target
Requires=docker.service
Wants=network-online.target
[Service]
TimeoutStartSec=0
Restart=always
ExecStartPre=-/usr/bin/docker stop {{ toolshed_backend_container }}
ExecStartPre=-/usr/bin/docker rm {{ toolshed_backend_container }}
ExecStart=/usr/bin/docker run --rm --name {{ toolshed_backend_container }} \
--user {{ toolshed_user.uid }}:{{ toolshed_user.group }} \
--env-file {{ toolshed_data_dir }}/backend.env \
-v {{ toolshed_data_dir }}:/data \
-p 127.0.0.1:{{ toolshed_backend_port }}:8000 \
{{ toolshed_backend_image }}:latest
ExecStop=/usr/bin/docker stop {{ toolshed_backend_container }}
[Install]
WantedBy=multi-user.target
notify: restart backend
# Installed before the bootstrap nginx flush_handlers below (which
# flushes every pending handler, not just reload nginx) - otherwise a
# fresh host would flush "restart backend" before this unit file exists
# and fail with "Could not find the requested service".
- name: Ensure toolshed-backend service is enabled and started
ansible.builtin.systemd:
name: toolshed-backend
daemon_reload: true
enabled: true
state: started
# The next few tasks build the frontend on the controller instead of the
# target host: `npm run build` pulls in bootstrap+jquery+vue+moment+
# js-nacl+qrcode, and esbuild's rendering/minification pass for that
# bundle needs more memory than small/memory-constrained target hosts
# (e.g. LXC containers without usable swap) reliably have. Only the
# resulting static dist/ is shipped to the target - the docker image
# itself never runs there. This assumes docker is already usable on the
# controller (not managed by this playbook, since "Install docker.io,
# nginx and rsync" above targets the remote host only).
- name: Checkout toolshed source (controller, for frontend build)
ansible.builtin.git:
repo: "{{ toolshed_repo_url | mandatory('toolshed_repo_url must be set as a host_var for ' ~ inventory_hostname) }}"
dest: "{{ toolshed_frontend_build_src_dir }}"
version: "{{ toolshed_version | default('stable') }}"
force: true
recursive: false
register: toolshed_frontend_checkout
delegate_to: localhost
become: false
- name: Build frontend builder docker image (controller)
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
--build-arg GIT_COMMIT={{ toolshed_frontend_checkout.after[:7] }}
-f {{ toolshed_frontend_build_src_dir }}/deploy/prod/Dockerfile.frontend {{ toolshed_frontend_build_src_dir }}/frontend
changed_when: true
delegate_to: localhost
become: false
- name: Create local frontend dist scratch directory (controller)
ansible.builtin.file:
path: "{{ toolshed_frontend_build_dist_dir }}"
state: directory
mode: "0755"
delegate_to: localhost
become: false
- name: Run frontend builder once to export the static build (controller)
ansible.builtin.command:
cmd: docker run --rm -v {{ toolshed_frontend_build_dist_dir }}:/output {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
changed_when: true
delegate_to: localhost
become: false
- name: Sync built frontend dist to the target host
ansible.posix.synchronize:
src: "{{ toolshed_frontend_build_dist_dir }}/"
dest: "{{ toolshed_dist_dir }}/"
delete: true
delegate_to: localhost
become: false
- name: Fix ownership of exported frontend build
ansible.builtin.file:
path: "{{ toolshed_dist_dir }}"
owner: www-data
group: www-data
recurse: true
- name: Create wiki static output directory
ansible.builtin.file:
path: "{{ toolshed_wiki_dist_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
- name: Build wiki builder docker image
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
-f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.wiki {{ toolshed_src_dir }}
changed_when: true
- name: Run wiki builder once to export the static site
ansible.builtin.command:
cmd: docker run --rm -v {{ toolshed_wiki_dist_dir }}:/output {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
changed_when: true
- name: Fix ownership of exported wiki build
ansible.builtin.file:
path: "{{ toolshed_wiki_dist_dir }}"
owner: www-data
group: www-data
recurse: true
- name: Create local fixtures directory
ansible.builtin.file:
path: "{{ toolshed_local_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
- name: Write registration domain list fixture
ansible.builtin.copy:
dest: "{{ toolshed_local_dir }}/domains.json"
owner: www-data
group: www-data
mode: "0644"
content: "{{ toolshed_register_domains | to_nice_json }}"
- name: Write DoH resolver fixture
ansible.builtin.copy:
dest: "{{ toolshed_local_dir }}/dns.json"
owner: www-data
group: www-data
mode: "0644"
content: "{{ toolshed_doh_resolvers | to_nice_json }}"
- name: Create ACME HTTP-01 challenge webroot
ansible.builtin.file:
path: "{{ toolshed_letsencrypt_webroot }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
when: not (behind_tls_proxy | default(false) | bool)
- name: Check for an existing Let's Encrypt certificate
ansible.builtin.stat:
path: "/etc/letsencrypt/live/{{ toolshed_hostnames[0] }}/fullchain.pem"
register: toolshed_cert
when: not (behind_tls_proxy | default(false) | bool)
- name: Configure nginx site for toolshed (bootstrap)
ansible.builtin.copy:
dest: /etc/nginx/sites-available/toolshed.conf
owner: root
group: root
mode: "0644"
content: "{{ toolshed_nginx_conf }}"
notify: reload nginx
- name: Remove default nginx site
ansible.builtin.file:
path: /etc/nginx/sites-enabled/default
state: absent
notify: reload nginx
- name: Enable toolshed nginx site
ansible.builtin.file:
src: /etc/nginx/sites-available/toolshed.conf
dest: /etc/nginx/sites-enabled/toolshed.conf
state: link
notify: reload nginx
# Certbot's webroot check (below) needs nginx already serving the
# bootstrap config from the tasks above, so force the reload now
# instead of waiting for the end of the play.
- name: Apply the bootstrap nginx config now
ansible.builtin.meta: flush_handlers
- name: Ensure the certbot renewal deploy-hook directory exists
ansible.builtin.file:
path: /etc/letsencrypt/renewal-hooks/deploy
state: directory
mode: "0755"
when: not (behind_tls_proxy | default(false) | bool)
- name: Reload nginx after certbot renews a certificate
ansible.builtin.copy:
dest: /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
owner: root
group: root
mode: "0755"
content: |
#!/bin/sh
systemctl reload nginx
when: not (behind_tls_proxy | default(false) | bool)
- name: Obtain or renew the Let's Encrypt certificate
ansible.builtin.command:
cmd: >-
certbot certonly --webroot -w {{ toolshed_letsencrypt_webroot }}
-d {{ toolshed_hostnames | join(' -d ') }}
--non-interactive --agree-tos
-m {{ toolshed_letsencrypt_email | mandatory('toolshed_letsencrypt_email must be set as a host_var for ' ~ inventory_hostname ~ ' since behind_tls_proxy is false there') }}
register: toolshed_certbot
changed_when: "'Certificate not yet due for renewal' not in toolshed_certbot.stdout"
when: not (behind_tls_proxy | default(false) | bool)
- name: Re-check the certificate now that certbot has run
ansible.builtin.stat:
path: "/etc/letsencrypt/live/{{ toolshed_hostnames[0] }}/fullchain.pem"
register: toolshed_cert
when: not (behind_tls_proxy | default(false) | bool)
- name: Configure nginx site for toolshed (final)
ansible.builtin.copy:
dest: /etc/nginx/sites-available/toolshed.conf
owner: root
group: root
mode: "0644"
content: "{{ toolshed_nginx_conf }}"
notify: reload nginx
handlers:
- name: validate nginx config
ansible.builtin.command: nginx -t
listen: reload nginx
changed_when: false
- name: reload nginx
ansible.builtin.systemd:
name: nginx
state: reloaded
listen: reload nginx
- name: restart backend
ansible.builtin.systemd:
name: toolshed-backend
daemon_reload: true
state: restarted
listen: restart backend