j3d1/toolshed
2
1
Fork 0
Code Issues 9 Pull requests 1 Projects Releases Packages Wiki Activity

stash

Browse source
This commit is contained in:
j3d1 2026-08-17 01:45:14 +02:00
parent 0f51f6e33f
commit 419893d93d
11 changed files with 925 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

13
backend/Dockerfile
View file

@ -1,13 +0,0 @@
FROM python:alpine
WORKDIR /app
RUN apk add --no-cache gcc musl-dev python3-dev
COPY requirements.txt /app
RUN pip install --upgrade pip && pip install -r requirements.txt
COPY . /app
RUN python configure.py
RUN python manage.py collectstatic --noinput
CMD python manage.py migrate && python manage.py runserver 0.0.0.0:8000 --insecure
# TODO serve static files with nginx and remove --insecure
EXPOSE 8000

39
backend/configure.py
View file

@ -32,25 +32,41 @@ def yesno(prompt, default=False):
def configure():
# Keys this function may generate/update, tracked so that if .env turns
# out to be unwritable (e.g. a prod container running as an unprivileged
# user, whose real config comes from --env-file instead) we can still
# print the resulting configuration for the operator to apply manually,
# rather than silently discarding it or crashing.
tracked_keys = ['SECRET_KEY', 'ALLOWED_HOSTS']
unwritable = False
if not os.path.exists('.env'):
if not yesno("the .env file does not exist, do you want to create it?", default=True):
print('Aborting')
exit(0)
if yesno("the .env file does not exist, do you want to create it?", default=True):
if not os.path.exists('.env.dist'):
print('No .env.dist file found')
exit(1)
else:
for key in dotenv.dotenv_values('.env.dist'):
if key not in tracked_keys:
tracked_keys.append(key)
from shutil import copyfile
try:
copyfile('.env.dist', '.env')
except PermissionError:
unwritable = True
env = dotenv.load_dotenv('.env')
if not env or not os.getenv('SECRET_KEY'):
dotenv.load_dotenv('.env')
if not os.getenv('SECRET_KEY'):
from django.core.management.utils import get_random_secret_key
print('No SECRET_KEY found in .env file, generating one...')
secret_key = get_random_secret_key()
os.environ['SECRET_KEY'] = secret_key
try:
with open('.env', 'a') as f:
f.write('\nSECRET_KEY=')
f.write(get_random_secret_key())
f.write(secret_key)
f.write('\n')
except PermissionError:
unwritable = True
# TODO rename ALLOWED_HOSTS to something more self-explanatory
current_hosts = os.getenv('ALLOWED_HOSTS')
@ -59,7 +75,16 @@ def configure():
if yesno("Do you want to add ALLOWED_HOSTS?"):
hosts = input("Enter a comma-separated list of allowed hosts: ")
joined_hosts = current_hosts + ',' + hosts if current_hosts else hosts
os.environ['ALLOWED_HOSTS'] = joined_hosts
try:
dotenv.set_key('.env', 'ALLOWED_HOSTS', joined_hosts)
except PermissionError:
unwritable = True
if unwritable:
print('Could not write .env (read-only working directory) - resulting configuration:')
for key in tracked_keys:
print('{}={}'.format(key, os.getenv(key, '')))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
import django

56
deploy/docker-compose.override.yml → deploy/dev/docker-compose.yml
View file

@ -1,49 +1,49 @@
version: '3.8'
name: deploy
name: dev
services:
backend-a:
build:
context: ../backend/
context: ../../backend/
dockerfile: ../deploy/dev/Dockerfile.backend
environment:
TOOLSHED_DB_PATH: /mnt/db.sqlite3
TOOLSHED_USERFILES_PATH: /mnt/userfiles
TOOLSHED_SETUP_PATH: /mnt/testdata.py
volumes:
- ../backend:/code
- ../deploy/dev/instance_a/a.env:/code/.env
- ../deploy/dev/instance_a/testdata.py:/mnt/testdata.py
- ../deploy/dev/instance_a/a.sqlite3:/mnt/db.sqlite3
- ../deploy/dev/instance_a/userfiles:/mnt/userfiles
- ../../backend:/code
- ./instance_a/a.env:/code/.env
- ./instance_a/testdata.py:/mnt/testdata.py
- ./instance_a/a.sqlite3:/mnt/db.sqlite3
- ./instance_a/userfiles:/mnt/userfiles
expose:
- 8000
command: bash -c "python configure.py; python configure.py testdata; python manage.py runserver 0.0.0.0:8000 --insecure"
backend-b:
build:
context: ../backend/
context: ../../backend/
dockerfile: ../deploy/dev/Dockerfile.backend
environment:
TOOLSHED_DB_PATH: /mnt/db.sqlite3
TOOLSHED_USERFILES_PATH: /mnt/userfiles
TOOLSHED_SETUP_PATH: /mnt/testdata.py
volumes:
- ../backend:/code
- ../deploy/dev/instance_b/b.env:/code/.env
- ../deploy/dev/instance_b/testdata.py:/mnt/testdata.py
- ../deploy/dev/instance_b/b.sqlite3:/mnt/db.sqlite3
- ../deploy/dev/instance_b/userfiles:/mnt/userfiles
- ../../backend:/code
- ./instance_b/b.env:/code/.env
- ./instance_b/testdata.py:/mnt/testdata.py
- ./instance_b/b.sqlite3:/mnt/db.sqlite3
- ./instance_b/userfiles:/mnt/userfiles
expose:
- 8000
command: bash -c "python configure.py; python configure.py testdata; python manage.py runserver 0.0.0.0:8000 --insecure"
frontend:
build:
context: ../frontend/
context: ../../frontend/
dockerfile: ../deploy/dev/Dockerfile.frontend
volumes:
- ../frontend:/app
- ../../frontend:/app
- /app/node_modules
expose:
- 5173
@ -51,11 +51,11 @@ services:
wiki:
build:
context: ../
context: ../../
dockerfile: deploy/dev/Dockerfile.wiki
volumes:
- ../mkdocs.yml:/wiki/mkdocs.yml
- ../docs:/wiki/docs
- ../../mkdocs.yml:/wiki/mkdocs.yml
- ../../docs:/wiki/docs
expose:
- 8001
command: mkdocs serve --dev-addr=0.0.0.0:8001
@ -63,12 +63,12 @@ services:
proxy-a:
build:
context: ./
dockerfile: dev/Dockerfile.proxy
dockerfile: Dockerfile.proxy
volumes:
- ./dev/instance_a/nginx-a.dev.conf:/etc/nginx/nginx.conf:ro
- ./dev/instance_a/dns.json:/var/www/dns.json:ro
- ./dev/instance_a/domains.json:/var/www/domains.json:ro
- ./dev/instance_a/userfiles:/var/www/userfiles:ro
- ./instance_a/nginx-a.dev.conf:/etc/nginx/nginx.conf:ro
- ./instance_a/dns.json:/var/www/dns.json:ro
- ./instance_a/domains.json:/var/www/domains.json:ro
- ./instance_a/userfiles:/var/www/userfiles:ro
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.3:5353:5353"
@ -76,19 +76,19 @@ services:
proxy-b:
build:
context: ./
dockerfile: dev/Dockerfile.proxy
dockerfile: Dockerfile.proxy
volumes:
- ./dev/instance_b/nginx-b.dev.conf:/etc/nginx/nginx.conf:ro
- ./dev/instance_b/userfiles:/var/www/userfiles:ro
- ./instance_b/nginx-b.dev.conf:/etc/nginx/nginx.conf:ro
- ./instance_b/userfiles:/var/www/userfiles:ro
ports:
- "127.0.0.2:8080:8080"
dns:
build:
context: ./dev/
context: ./
dockerfile: Dockerfile.dns
volumes:
- ./dev/zone.json:/dns/zone.json
- ./zone.json:/dns/zone.json
expose:
- 8053
networks:

2
deploy/prod/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.secrets/
inventory.yml

14
deploy/prod/Dockerfile
View file

@ -1,14 +0,0 @@
FROM node:alpine as builder
WORKDIR /app
COPY ./package.json /app/package.json
COPY . /app
RUN npm install
RUN npm run build
FROM nginx:alpine as runner
RUN apk add --update npm
WORKDIR /app
COPY --from=builder /app/dist /usr/share/nginx/html
COPY ./nginx.conf /etc/nginx/nginx.conf
EXPOSE 80

27
deploy/prod/Dockerfile.backend Normal file
View file

@ -0,0 +1,27 @@
# Production image for the Django backend.
# Runs migrations then serves the app with gunicorn on port 8000.
# Static files are collected at build time into /app/staticfiles and
# served by the backend itself behind the host nginx reverse proxy.
FROM python:3.11-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
DJANGO_SETTINGS_MODULE=backend.settings
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r requirements.txt gunicorn
COPY . .
# collectstatic only needs Django settings to import cleanly, not a real
# secret; the actual SECRET_KEY is injected at container runtime via
# --env-file and overrides this.
RUN SECRET_KEY=build-time-placeholder python manage.py collectstatic --noinput
EXPOSE 8000
CMD ["sh", "-c", "python manage.py migrate --noinput && exec gunicorn backend.wsgi:application --bind 0.0.0.0:8000 --workers 3"]

18
deploy/prod/Dockerfile.frontend Normal file
View file

@ -0,0 +1,18 @@
# Build-only image for the Vue frontend.
# It is never run as a service: ansible builds this image once, runs it
# with the host output directory bind-mounted at /output, the container
# copies the compiled static build into it, and exits. Nginx on the host
# then serves that directory directly.
FROM node:20-alpine AS build
WORKDIR /app
COPY package.json package-lock.json ./
COPY extras/ ./extras/
RUN npm ci
COPY . .
RUN npm run build
FROM alpine AS export
COPY --from=build /app/dist /dist
VOLUME /output
CMD ["sh", "-c", "rm -rf /output/* && cp -a /dist/. /output/"]

17
deploy/prod/Dockerfile.wiki Normal file
View file

@ -0,0 +1,17 @@
# Build-only image for the project wiki (mkdocs).
# It is never run as a service: ansible builds this image once, runs it
# with the host output directory bind-mounted at /output, the container
# copies the built static site into it, and exits. Nginx on the host
# then serves that directory directly, the same way it does the frontend.
FROM python:3.11-slim AS build
WORKDIR /wiki
RUN pip install --no-cache-dir mkdocs
COPY mkdocs.yml ./
COPY docs/ ./docs/
RUN mkdocs build
FROM alpine AS export
COPY --from=build /wiki/site /site
VOLUME /output
CMD ["sh", "-c", "rm -rf /output/* && cp -a /site/. /output/"]

206
deploy/prod/README.md Normal file
View file

@ -0,0 +1,206 @@
# Toolshed production deployment — manual steps
`playbook.yml` automates installing docker.io and nginx (plus certbot, and
obtaining/renewing a TLS certificate with it, on hosts that manage their own
— see `behind_tls_proxy` below), building the backend, frontend and wiki
images, exporting the frontend and wiki static builds for nginx to serve,
writing the small `/local/domains` and `/local/dns` fixture files the
frontend fetches directly (registration domain list and DoH resolver
preference — see `toolshed_register_domains`/`toolshed_doh_resolvers` in
`playbook.yml`), configuring nginx, and installing the `toolshed-backend`
systemd service. It does **not** set up the target server or DNS. Those are
manual, one-time steps and are covered here. Seeding the backend's shared
reference data is also a manual, one-time step — see
[First superuser & shared reference data](#5-first-superuser--shared-reference-data).
## 1. Server & firewall
- A Debian/Ubuntu host reachable over SSH.
- Copy `inventory.example.yml` to `inventory.yml` (git-ignored, since it
holds real hostnames/IPs) and fill in your host(s) — see
[Per-deployment configuration](#2-per-deployment-configuration).
- Inbound TCP 80 open in the firewall/security group. Also open 443 unless
`behind_tls_proxy: true` — and keep both open permanently, not just for the
initial deploy: certbot's renewal timer needs 80 for the ACME HTTP-01
challenge and 443 for HTTPS traffic for as long as this host is live.
## 2. Per-deployment configuration
Each entry under `hosts:` in `inventory.yml` is its own independent
deployment (its own repo checkout, database, domain, systemd service and
Django `SECRET_KEY` — nothing is shared between hosts). Set these as
host_vars directly on each host entry, not via `-e` on the command line,
so a single `inventory.yml` can hold several unrelated deployments safely:
```yaml
toolshed:
hosts:
my-server:
ansible_host: 203.0.113.10
ansible_user: deploy
toolshed_domain: toolshed.webdomain.tld
toolshed_handle_domain: yourtoolshed.tld # optional, see below
toolshed_repo_url: git@example.com:your-org/toolshed.git
behind_tls_proxy: false
```
- `toolshed_domain` — the **web domain**: the nginx `server_name`, Django
`ALLOWED_HOSTS`, and the hostname you'll point a TLS cert at — e.g.
`toolshed.webdomain.tld`. Required, no default. This is not necessarily the
same as the **handle domain** your users log in with (the part after `@`
in `user@yourtoolshed.tld`) — see [DNS](#3-dns) for how those two relate.
- `toolshed_handle_domain` — the **handle domain**, only needed when it's
different from `toolshed_domain`. Omit it when the two are the same (it
then defaults to `toolshed_domain`). Set so nginx/Django accept requests
for either domain, whichever ends up as the `Host` header.
- `toolshed_repo_url` — the git remote the playbook checks out and builds
from. Required, no default.
- `toolshed_version` — the branch, tag or commit to check out and build.
Optional, defaults to `stable`.
- `behind_tls_proxy` — `true` if TLS for this host is already terminated by
something in front of it (e.g. an external reverse proxy or load
balancer) that forwards plain HTTP here; `false` if this nginx has to
terminate TLS itself. This controls two things:
- Whether nginx trusts an upstream `X-Forwarded-Proto` header or sets its
own — get this wrong and Django's `SECURE_PROXY_SSL_HEADER` check
(`backend/backend/settings.py`) will treat every request as insecure or,
flipped the other way, treat plain HTTP as secure.
- Whether the playbook manages TLS at all. When `false`, it automatically
obtains a Let's Encrypt certificate via certbot and switches nginx over
to it — nothing to do manually beyond DNS (below). certbot's own systemd
timer keeps renewing it afterwards, independent of the playbook.
- `toolshed_letsencrypt_email` — required whenever `behind_tls_proxy` is
`false`; the account email certbot registers the certificate under
(used only for renewal-failure notices). Ignored otherwise.
- `http_port` — optional, defaults to `80`. Only relevant when
`behind_tls_proxy: true` and whatever's in front of this host forwards to
a nonstandard port instead of 80.
- `doh_resolvers` — optional, defaults to `["1.1.1.1", "8.8.8.8"]` (the same
hardcoded fallback the frontend itself uses, see `frontend/src/dns.js`).
DNS-over-HTTPS resolvers the frontend uses to look up a handle domain's
`_toolshed-server._tcp` SRV record before it has a cached preference.
Written to `/local/dns` at deploy time; only worth overriding as a
host_var (or `-e doh_resolvers='["9.9.9.9"]'`) if you want this
deployment to prefer a specific resolver.
## 3. DNS
There are two distinct domains at play here, and it's easy to conflate them:
- **Web domain** — the machine's actual hostname: nginx `server_name`,
Django `ALLOWED_HOSTS`, your TLS cert, what's in `toolshed_domain`. This is
what an A/AAAA record has to resolve to the server's IP for.
- **Handle domain** — the part after the `@` in a username, e.g.
`user@yourtoolshed.tld`. Toolshed usernames don't encode a server address
directly; the frontend resolves the handle domain to a server via an SRV
record, `_toolshed-server._tcp.<handle domain>.` (see
`frontend/src/store.js`, `lookupServer`). What's in `toolshed_handle_domain`
(see [Per-deployment configuration](#2-per-deployment-configuration)) only
makes nginx/Django accept it as a `Host` header — publishing the actual SRV
record is still a separate, manual DNS step, covered below.
The SRV lookup happens for every login, not just federation with other
servers, so **every** deployment needs it published for its own handle
domain — even a standalone server that only ever serves itself.
These two domains can be **the same** or **completely different**, and
that's exactly the choice between an A record and an SRV record:
- **Same domain**: if `yourtoolshed.tld` is both the web domain and the
handle domain, it needs both an A record (so the domain itself resolves to
the server) and an SRV record that happens to point back at itself.
- **Different domains**: the handle domain only needs the SRV record — no A
record of its own — pointing at whatever web domain the server actually
lives at. This is useful when the handle you give out (short, brandable,
independent of hosting) shouldn't have to match wherever the box is
actually deployed (a subdomain of a shared hosting provider, an internal
service name, etc.).
**a) A/AAAA record — web domain → server IP:**
```sh
dig <your-web-domain> A
```
**b) SRV record — handle domain → web domain + port.** Use port 443: the
federation protocol is HTTPS-only.
```sh
dig _toolshed-server._tcp.<your-handle-domain> SRV
```
For example, with a handle domain of `yourtoolshed.tld` and a web domain of
`toolshed.webdomain.tld`:
```
$ dig _toolshed-server._tcp.yourtoolshed.tld srv
_toolshed-server._tcp.yourtoolshed.tld. 300 IN SRV 10 10 443 toolshed.webdomain.tld.
$ dig toolshed.webdomain.tld A
toolshed.webdomain.tld. 300 IN A 203.0.113.10
```
If you instead want `yourtoolshed.tld` itself to be the web domain too, its
SRV record just points at itself (`... SRV 10 10 443 yourtoolshed.tld.`) and
it additionally needs its own A record.
## 4. Secrets
`toolshed_secret_key` is generated once per host by the playbook (via the
`password` lookup, keyed by the host's inventory name) and stored as
`.secrets/<inventory-hostname>_secret_key` on the *control* machine, not on
the target. Back these files up — losing one invalidates all sessions and
signed cookies for that deployment on its next redeploy. They're git-ignored
on purpose; never commit them.
## 5. First superuser & shared reference data
The production backend image only runs `migrate` and `collectstatic` at
startup (see `Dockerfile.backend`) — unlike the dev compose setup, it never
runs the interactive `configure.py`. Two things dev gets "for free" from that
script therefore need doing manually, once, after a host's backend container
is first up (run these on the target host itself, or prefix with
`ssh <that-host>`):
- **Superuser account:**
```sh
docker exec -it toolshed-backend python manage.py createsuperuser
```
- **Shared reference data** (the standard categories/properties/tags
shipped in `backend/shared_data/*.json` — tools, electrical, screws, IT,
etc.): without this step a fresh deployment starts with none of them.
Run `configure.py` interactively (the `-it` flags matter — the script's
prompts only appear with a real tty) and answer "yes" when it asks to
import them:
```sh
docker exec -it toolshed-backend python configure.py
```
The other prompts it asks first (create `.env`, create a database) are
harmless to answer "yes" to as well: the container already gets its real
`SECRET_KEY`/`ALLOWED_HOSTS`/db path from the environment (the systemd unit
passes them via `--env-file`, see the "Write backend environment file" task
in `playbook.yml`), those checks just look for files at paths relative to
`/app` that don't exist in this container, and re-running `migrate` against
the real database is idempotent. You can say "no" to the superuser prompt
here if you already created one above.
## 6. Running the playbook
Always target one host at a time with `--limit` — running against the whole
`toolshed` group in one invocation would apply every host's own
`toolshed_domain`/`toolshed_repo_url` correctly (they're per-host vars, see
[Per-deployment configuration](#2-per-deployment-configuration)), but rolls
out all deployments back-to-back in one run, which is rarely what you want:
```sh
ansible-playbook -i inventory.yml playbook.yml --limit my-server
```
Re-run it to roll out a new version to that host. It deploys whatever
`toolshed_version` is set for that host (`stable` by default) — set the
host_var for a persistent change, or pass `-e toolshed_version=<branch/tag/commit>`
for a one-off deploy of something else.

42
deploy/prod/inventory.example.yml Normal file
View file

@ -0,0 +1,42 @@
---
# Copy this file to inventory.yml (git-ignored) and fill in your real
# hosts. Each entry under hosts: is an independent deployment - see the
# README's "Per-deployment configuration" section for what each var means.
toolshed:
hosts:
my-server:
ansible_host: 203.0.113.10
ansible_user: deploy
# toolshed_domain is the "web domain" - see the README's DNS section
# for how this relates to the separate "handle domain" your users
# log in with (user@yourtoolshed.tld).
toolshed_domain: toolshed.webdomain.tld
# Optional - only needed if the handle domain differs from the web
# domain above. Omit it entirely when they're the same.
toolshed_handle_domain: yourtoolshed.tld
toolshed_repo_url: git@example.com:your-org/toolshed.git
# Optional - branch, tag or commit to deploy. Defaults to "stable".
toolshed_version: stable
# true if something in front of this host already terminates TLS
# (reverse proxy/load balancer), false if this nginx must do it itself.
behind_tls_proxy: false
# Required whenever behind_tls_proxy is false: the playbook obtains
# its own Let's Encrypt certificate via certbot, which needs an
# account email for renewal notices.
toolshed_letsencrypt_email: admin@example.com
# A second, unrelated deployment behind an existing TLS-terminating
# proxy - remove this if you only run one instance. Here the handle
# domain and web domain are the same, so toolshed_handle_domain is
# simply omitted, and toolshed_letsencrypt_email isn't needed since
# this nginx never handles TLS itself.
my-other-server:
ansible_host: my-other-server.example.com
ansible_user: deploy
toolshed_domain: toolshed.example.com
toolshed_repo_url: git@example.com:your-org/toolshed.git
behind_tls_proxy: true
# Only needed if the proxy in front forwards to something other than
# port 80 on this host.
http_port: 8080

543
deploy/prod/playbook.yml Normal file
View file

@ -0,0 +1,543 @@
---
# Production deploy for toolshed.
#
# - installs docker.io and nginx on the target (plus certbot, unless
# behind_tls_proxy is true)
# - checks out the source and builds the backend and frontend docker images
# - runs the frontend image once to export its static build, which nginx
# then serves directly (the frontend image is never run as a service)
# - configures nginx (inline template, no separate .conf file) and, unless
# behind_tls_proxy is true, obtains/renews a Let's Encrypt certificate via
# certbot and switches nginx over to it automatically - no manual TLS step
# - installs and manages a systemd service that runs the backend container
#
# Usage (each host is its own independent deployment - always target one
# at a time, never the whole "toolshed" group in one run):
# ansible-playbook -i inventory.yml playbook.yml --limit my-server
#
# toolshed_repo_url, toolshed_domain, toolshed_handle_domain (optional),
# toolshed_version (optional, defaults to "stable"), behind_tls_proxy and
# toolshed_letsencrypt_email (required unless behind_tls_proxy is true) are
# per-deployment and must be set as host_vars in inventory.yml (copy
# inventory.example.yml) rather than here or via -e, so that each host in
# the "toolshed" group can point at its own repo/domain/branch. They're read
# with `mandatory`/`default()` below instead of being declared in play
# `vars:`, since play vars always take precedence over inventory host_vars
# and would otherwise silently override whatever is set per-host.
- name: Deploy toolshed
hosts: toolshed
become: true
vars:
toolshed_src_dir: /opt/toolshed/src
toolshed_data_dir: /opt/toolshed/data
toolshed_dist_dir: /var/www/toolshed
toolshed_backend_image: toolshed-backend
toolshed_frontend_image: toolshed-frontend-builder
toolshed_wiki_image: toolshed-wiki-builder
toolshed_backend_container: toolshed-backend
toolshed_backend_port: 8000
toolshed_wiki_dist_dir: /var/www/toolshed-wiki
toolshed_local_dir: /var/www/toolshed-local
# Domain(s) this server accepts registrations for (the "handle domain" -
# see the README's DNS section). Served as a static /local/domains
# fixture that the frontend's registration/pairing forms fetch to
# populate their domain dropdown (frontend/src/views/Register.vue,
# Pairing.vue) - without it that dropdown is just empty.
toolshed_register_domains: "{{ [toolshed_handle_domain | default(toolshed_domain)] | unique }}"
# DoH resolvers the frontend falls back to for SRV lookups when it has
# no cached preference yet, served as a static /local/dns fixture. These
# match the frontend's own hardcoded fallback (frontend/src/dns.js), so
# this mostly makes the choice explicit and per-host overridable (e.g.
# -e doh_resolvers='["9.9.9.9"]') rather than changing behavior.
toolshed_doh_resolvers: "{{ doh_resolvers | default(['1.1.1.1', '8.8.8.8']) }}"
# Docker tags can't contain "/", but toolshed_version is a git ref and
# branch names like "jedi/proto/frontend" do - sanitize before using it
# as an image tag. The raw value is still used as-is for the actual git
# checkout, where slashes are fine.
toolshed_image_tag: "{{ (toolshed_version | default('stable')) | replace('/', '-') }}"
toolshed_debug: "False"
# Plain HTTP listen port. Only relevant behind an external proxy that
# forwards to something other than 80 (see http_port in inventory.yml);
# when this nginx terminates TLS itself, the public port is always 443.
toolshed_http_port: "{{ http_port | default(80) }}"
toolshed_letsencrypt_webroot: /var/www/letsencrypt
# Nginx sets its own X-Forwarded-Proto from $scheme when it terminates
# TLS itself. Behind an external TLS-terminating proxy, $scheme at this
# nginx is always "http" (the proxy already stripped TLS one hop
# earlier), so overwriting the header with $scheme would tell Django
# every request is insecure. In that case pass through the proxy's own
# header instead.
toolshed_x_forwarded_proto: >-
{{ '$http_x_forwarded_proto' if (behind_tls_proxy | default(false) | bool) else '$scheme' }}
# The web domain (toolshed_domain, mandatory) and the handle domain
# (toolshed_handle_domain, optional - defaults to the web domain when
# they're the same) both need to be accepted by nginx/Django, since
# either may show up as the Host header depending on how the admin set
# up DNS for this deployment. Deduplicated so setting them equal
# doesn't produce a repeated entry.
toolshed_hostnames: >-
{{ [toolshed_domain | mandatory('toolshed_domain must be set as a host_var for ' ~ inventory_hostname),
toolshed_handle_domain | default(toolshed_domain)] | unique }}
# Generated once per host on the controller and reused on every
# subsequent run against that host, keyed by inventory_hostname so
# separate deployments never end up sharing a Django SECRET_KEY.
toolshed_secret_key: >-
{{ lookup('ansible.builtin.password',
playbook_dir ~ '/.secrets/' ~ inventory_hostname ~ '_secret_key length=64 chars=ascii_letters,digits') }}
# Rendered twice against the same var (see the tasks below): once before
# a certificate exists (serves the site plainly over toolshed_http_port,
# or over 80/plain-HTTP forever if behind_tls_proxy), and once after
# certbot has obtained one, at which point the plain HTTP vhost switches
# to a redirect and a 443 vhost with the real content appears. Whichever
# of those two states applies, toolshed_cert (a registered `stat` result,
# undefined/false until it's checked) decides which one renders - this
# is the "another nginx config" from a single inline template, driven by
# behind_tls_proxy and certificate state rather than a separate file.
toolshed_nginx_conf: |
upstream toolshed_backend {
server 127.0.0.1:{{ toolshed_backend_port }};
}
{% macro toolshed_locations() %}
location /api {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /auth {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /media {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /djangoadmin {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /docs {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
proxy_pass http://toolshed_backend;
}
location /static {
proxy_pass http://toolshed_backend/static;
}
location /wiki/ {
alias {{ toolshed_wiki_dist_dir }}/;
try_files $uri $uri/ =404;
}
location = /wiki {
return 301 /wiki/;
}
# Static fixtures the frontend fetches directly (registration
# domain list, DoH resolver preference) - see toolshed_register_domains
# and toolshed_doh_resolvers above.
location /local/ {
alias {{ toolshed_local_dir }}/;
try_files $uri.json =404;
add_header Content-Type application/json;
}
# Vue-router history mode: fall back to index.html for
# any path that isn't a real static file.
location / {
try_files $uri $uri/ /index.html;
}
{% endmacro %}
{% if behind_tls_proxy | default(false) | bool %}
server {
listen {{ toolshed_http_port }};
listen [::]:{{ toolshed_http_port }};
server_name {{ toolshed_hostnames | join(' ') }};
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
}
{% else %}
{% set tls_active = toolshed_cert.stat.exists | default(false) %}
server {
listen {{ toolshed_http_port }};
listen [::]:{{ toolshed_http_port }};
server_name {{ toolshed_hostnames | join(' ') }};
location /.well-known/acme-challenge/ {
root {{ toolshed_letsencrypt_webroot }};
}
{% if tls_active %}
location / {
return 301 https://$host$request_uri;
}
{% else %}
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
{% endif %}
}
{% if tls_active %}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name {{ toolshed_hostnames | join(' ') }};
ssl_certificate /etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ toolshed_domain }}/privkey.pem;
client_max_body_size 128M;
root {{ toolshed_dist_dir }};
index index.html;
{{ toolshed_locations() }}
}
{% endif %}
{% endif %}
tasks:
- name: Install docker.io and nginx
ansible.builtin.apt:
name:
- docker.io
- nginx
state: present
update_cache: true
- name: Install certbot
ansible.builtin.apt:
name: certbot
state: present
when: not (behind_tls_proxy | default(false) | bool)
- name: Ensure docker is running and enabled
ansible.builtin.systemd:
name: docker
state: started
enabled: true
- name: Ensure nginx is running and enabled
ansible.builtin.systemd:
name: nginx
state: started
enabled: true
- name: Checkout toolshed source
ansible.builtin.git:
repo: "{{ toolshed_repo_url | mandatory('toolshed_repo_url must be set as a host_var for ' ~ inventory_hostname) }}"
dest: "{{ toolshed_src_dir }}"
version: "{{ toolshed_version | default('stable') }}"
force: true
# frontend/extras is registered as a submodule but unused and its
# pinned commit isn't fetchable from upstream - don't let a broken
# submodule block the checkout.
recursive: false
- name: Create toolshed system user
ansible.builtin.user:
name: toolshed
system: true
shell: /usr/sbin/nologin
home: "{{ toolshed_data_dir }}"
create_home: false
register: toolshed_user
- name: Create backend data directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: toolshed
group: toolshed
mode: "0750"
loop:
- "{{ toolshed_data_dir }}"
- "{{ toolshed_data_dir }}/userfiles"
- name: Create frontend static output directory
ansible.builtin.file:
path: "{{ toolshed_dist_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
- name: Write backend environment file
ansible.builtin.copy:
dest: "{{ toolshed_data_dir }}/backend.env"
# Root-owned and unreadable by the toolshed user on purpose: this is
# read by the docker daemon (root) via --env-file at container
# start and injected directly as env vars, so the containerized app
# - which runs as the toolshed user, see the systemd unit below -
# never needs filesystem access to its own SECRET_KEY.
owner: root
group: root
mode: "0600"
content: |
DEBUG={{ toolshed_debug }}
SECRET_KEY={{ toolshed_secret_key }}
ALLOWED_HOSTS={{ toolshed_hostnames | join(',') }}
SERVE_X_ACCEL_REDIRECT=False
TOOLSHED_DB_PATH=/data/db.sqlite3
TOOLSHED_USERFILES_PATH=/data/userfiles
notify: restart backend
- name: Build backend docker image
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_backend_image }}:{{ toolshed_image_tag }}
-f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.backend {{ toolshed_src_dir }}/backend
changed_when: true
notify: restart backend
- name: Tag backend image as latest
ansible.builtin.command:
cmd: docker tag {{ toolshed_backend_image }}:{{ toolshed_image_tag }} {{ toolshed_backend_image }}:latest
changed_when: true
notify: restart backend
- name: Install systemd unit for the backend container
ansible.builtin.copy:
dest: /etc/systemd/system/toolshed-backend.service
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=Toolshed backend (Django) container
After=docker.service network-online.target
Requires=docker.service
Wants=network-online.target
[Service]
TimeoutStartSec=0
Restart=always
ExecStartPre=-/usr/bin/docker stop {{ toolshed_backend_container }}
ExecStartPre=-/usr/bin/docker rm {{ toolshed_backend_container }}
ExecStart=/usr/bin/docker run --rm --name {{ toolshed_backend_container }} \
--user {{ toolshed_user.uid }}:{{ toolshed_user.group }} \
--env-file {{ toolshed_data_dir }}/backend.env \
-v {{ toolshed_data_dir }}:/data \
-p 127.0.0.1:{{ toolshed_backend_port }}:8000 \
{{ toolshed_backend_image }}:latest
ExecStop=/usr/bin/docker stop {{ toolshed_backend_container }}
[Install]
WantedBy=multi-user.target
notify: restart backend
# Installed before the bootstrap nginx flush_handlers below (which
# flushes every pending handler, not just reload nginx) - otherwise a
# fresh host would flush "restart backend" before this unit file exists
# and fail with "Could not find the requested service".
- name: Ensure toolshed-backend service is enabled and started
ansible.builtin.systemd:
name: toolshed-backend
daemon_reload: true
enabled: true
state: started
- name: Build frontend builder docker image
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
-f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.frontend {{ toolshed_src_dir }}/frontend
changed_when: true
- name: Run frontend builder once to export the static build
ansible.builtin.command:
cmd: docker run --rm -v {{ toolshed_dist_dir }}:/output {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
changed_when: true
- name: Fix ownership of exported frontend build
ansible.builtin.file:
path: "{{ toolshed_dist_dir }}"
owner: www-data
group: www-data
recurse: true
- name: Create wiki static output directory
ansible.builtin.file:
path: "{{ toolshed_wiki_dist_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
- name: Build wiki builder docker image
ansible.builtin.command:
cmd: >-
docker build -t {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
-f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.wiki {{ toolshed_src_dir }}
changed_when: true
- name: Run wiki builder once to export the static site
ansible.builtin.command:
cmd: docker run --rm -v {{ toolshed_wiki_dist_dir }}:/output {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
changed_when: true
- name: Fix ownership of exported wiki build
ansible.builtin.file:
path: "{{ toolshed_wiki_dist_dir }}"
owner: www-data
group: www-data
recurse: true
- name: Create local fixtures directory
ansible.builtin.file:
path: "{{ toolshed_local_dir }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
- name: Write registration domain list fixture
ansible.builtin.copy:
dest: "{{ toolshed_local_dir }}/domains.json"
owner: www-data
group: www-data
mode: "0644"
content: "{{ toolshed_register_domains | to_nice_json }}"
- name: Write DoH resolver fixture
ansible.builtin.copy:
dest: "{{ toolshed_local_dir }}/dns.json"
owner: www-data
group: www-data
mode: "0644"
content: "{{ toolshed_doh_resolvers | to_nice_json }}"
- name: Create ACME HTTP-01 challenge webroot
ansible.builtin.file:
path: "{{ toolshed_letsencrypt_webroot }}"
state: directory
owner: www-data
group: www-data
mode: "0755"
when: not (behind_tls_proxy | default(false) | bool)
- name: Check for an existing Let's Encrypt certificate
ansible.builtin.stat:
path: "/etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem"
register: toolshed_cert
when: not (behind_tls_proxy | default(false) | bool)
- name: Configure nginx site for toolshed (bootstrap)
ansible.builtin.copy:
dest: /etc/nginx/sites-available/toolshed.conf
owner: root
group: root
mode: "0644"
content: "{{ toolshed_nginx_conf }}"
notify: reload nginx
- name: Remove default nginx site
ansible.builtin.file:
path: /etc/nginx/sites-enabled/default
state: absent
notify: reload nginx
- name: Enable toolshed nginx site
ansible.builtin.file:
src: /etc/nginx/sites-available/toolshed.conf
dest: /etc/nginx/sites-enabled/toolshed.conf
state: link
notify: reload nginx
# Certbot's webroot check (below) needs nginx already serving the
# bootstrap config from the tasks above, so force the reload now
# instead of waiting for the end of the play.
- name: Apply the bootstrap nginx config now
ansible.builtin.meta: flush_handlers
- name: Ensure the certbot renewal deploy-hook directory exists
ansible.builtin.file:
path: /etc/letsencrypt/renewal-hooks/deploy
state: directory
mode: "0755"
when: not (behind_tls_proxy | default(false) | bool)
- name: Reload nginx after certbot renews a certificate
ansible.builtin.copy:
dest: /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
owner: root
group: root
mode: "0755"
content: |
#!/bin/sh
systemctl reload nginx
when: not (behind_tls_proxy | default(false) | bool)
- name: Obtain or renew the Let's Encrypt certificate
ansible.builtin.command:
cmd: >-
certbot certonly --webroot -w {{ toolshed_letsencrypt_webroot }}
-d {{ toolshed_hostnames | join(' -d ') }}
--non-interactive --agree-tos
-m {{ toolshed_letsencrypt_email | mandatory('toolshed_letsencrypt_email must be set as a host_var for ' ~ inventory_hostname ~ ' since behind_tls_proxy is false there') }}
register: toolshed_certbot
changed_when: "'Certificate not yet due for renewal' not in toolshed_certbot.stdout"
when: not (behind_tls_proxy | default(false) | bool)
- name: Re-check the certificate now that certbot has run
ansible.builtin.stat:
path: "/etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem"
register: toolshed_cert
when: not (behind_tls_proxy | default(false) | bool)
- name: Configure nginx site for toolshed (final)
ansible.builtin.copy:
dest: /etc/nginx/sites-available/toolshed.conf
owner: root
group: root
mode: "0644"
content: "{{ toolshed_nginx_conf }}"
notify: reload nginx
handlers:
- name: validate nginx config
ansible.builtin.command: nginx -t
listen: reload nginx
changed_when: false
- name: reload nginx
ansible.builtin.systemd:
name: nginx
state: reloaded
listen: reload nginx
- name: restart backend
ansible.builtin.systemd:
name: toolshed-backend
daemon_reload: true
state: restarted
listen: restart backend