toolshed/deploy/prod/playbook.yml

---
# Production deploy for toolshed.
#
# - installs docker.io and nginx on the target (plus certbot, unless
#   behind_tls_proxy is true)
# - checks out the source and builds the backend and frontend docker images
# - runs the frontend image once to export its static build, which nginx
#   then serves directly (the frontend image is never run as a service)
# - configures nginx (inline template, no separate .conf file) and, unless
#   behind_tls_proxy is true, obtains/renews a Let's Encrypt certificate via
#   certbot and switches nginx over to it automatically - no manual TLS step
# - installs and manages a systemd service that runs the backend container
#
# Usage (each host is its own independent deployment - always target one
# at a time, never the whole "toolshed" group in one run):
#   ansible-playbook -i inventory.yml playbook.yml --limit my-server
#
# toolshed_repo_url, toolshed_domain, toolshed_handle_domain (optional),
# toolshed_version (optional, defaults to "stable"), behind_tls_proxy and
# toolshed_letsencrypt_email (required unless behind_tls_proxy is true) are
# per-deployment and must be set as host_vars in inventory.yml (copy
# inventory.example.yml) rather than here or via -e, so that each host in
# the "toolshed" group can point at its own repo/domain/branch. They're read
# with `mandatory`/`default()` below instead of being declared in play
# `vars:`, since play vars always take precedence over inventory host_vars
# and would otherwise silently override whatever is set per-host.

- name: Deploy toolshed
  hosts: toolshed
  become: true

  vars:
    toolshed_src_dir: /opt/toolshed/src
    toolshed_data_dir: /opt/toolshed/data
    toolshed_dist_dir: /var/www/toolshed

    toolshed_backend_image: toolshed-backend
    toolshed_frontend_image: toolshed-frontend-builder
    toolshed_wiki_image: toolshed-wiki-builder
    toolshed_backend_container: toolshed-backend
    toolshed_backend_port: 8000
    toolshed_wiki_dist_dir: /var/www/toolshed-wiki
    toolshed_local_dir: /var/www/toolshed-local
    # Django's collectstatic output (admin/drf-yasg assets etc.), exported
    # from the built backend image so nginx can serve it directly instead of
    # proxying to gunicorn for every asset request.
    toolshed_static_dir: /var/www/toolshed-static
    # Domain(s) this server accepts registrations for (the "handle domain" -
    # see the README's DNS section). Served as a static /local/domains
    # fixture that the frontend's registration/pairing forms fetch to
    # populate their domain dropdown (frontend/src/views/Register.vue,
    # Pairing.vue) - without it that dropdown is just empty.
    toolshed_register_domains: "{{ [toolshed_handle_domain | default(toolshed_domain)] | unique }}"
    # DoH resolvers the frontend falls back to for SRV lookups when it has
    # no cached preference yet, served as a static /local/dns fixture. These
    # match the frontend's own hardcoded fallback (frontend/src/dns.js), so
    # this mostly makes the choice explicit and per-host overridable (e.g.
    # -e doh_resolvers='["9.9.9.9"]') rather than changing behavior.
    toolshed_doh_resolvers: "{{ doh_resolvers | default(['1.1.1.1', '8.8.8.8']) }}"
    # Docker tags can't contain "/", but toolshed_version is a git ref and
    # branch names like "jedi/proto/frontend" do - sanitize before using it
    # as an image tag. The raw value is still used as-is for the actual git
    # checkout, where slashes are fine.
    toolshed_image_tag: "{{ (toolshed_version | default('stable')) | replace('/', '-') }}"

    toolshed_debug: "False"
    # Plain HTTP listen port. Only relevant behind an external proxy that
    # forwards to something other than 80 (see http_port in inventory.yml);
    # when this nginx terminates TLS itself, the public port is always 443.
    toolshed_http_port: "{{ http_port | default(80) }}"
    toolshed_letsencrypt_webroot: /var/www/letsencrypt
    # Nginx sets its own X-Forwarded-Proto from $scheme when it terminates
    # TLS itself. Behind an external TLS-terminating proxy, $scheme at this
    # nginx is always "http" (the proxy already stripped TLS one hop
    # earlier), so overwriting the header with $scheme would tell Django
    # every request is insecure. In that case pass through the proxy's own
    # header instead.
    toolshed_x_forwarded_proto: >-
      {{ '$http_x_forwarded_proto' if (behind_tls_proxy | default(false) | bool) else '$scheme' }}
    # Only the web domain (toolshed_domain) - nginx server_name, Django
    # ALLOWED_HOSTS, and the cert certbot requests. The handle domain
    # (toolshed_handle_domain) is resolved by clients via its own SRV record
    # and doesn't necessarily have an A record pointing at this host at all
    # (see the README's DNS section), so it can't reliably serve an HTTP-01
    # challenge or ever show up as this nginx's Host header.
    toolshed_hostnames: >-
      {{ [toolshed_domain | mandatory('toolshed_domain must be set as a host_var for ' ~ inventory_hostname)] }}
    # Generated once per host on the controller and reused on every
    # subsequent run against that host, keyed by inventory_hostname so
    # separate deployments never end up sharing a Django SECRET_KEY.
    toolshed_secret_key: >-
      {{ lookup('ansible.builtin.password',
                 playbook_dir ~ '/.secrets/' ~ inventory_hostname ~ '_secret_key length=64 chars=ascii_letters,digits') }}

    # Rendered twice against the same var (see the tasks below): once before
    # a certificate exists (serves the site plainly over toolshed_http_port,
    # or over 80/plain-HTTP forever if behind_tls_proxy), and once after
    # certbot has obtained one, at which point the plain HTTP vhost switches
    # to a redirect and a 443 vhost with the real content appears. Whichever
    # of those two states applies, toolshed_cert (a registered `stat` result,
    # undefined/false until it's checked) decides which one renders - this
    # is the "another nginx config" from a single inline template, driven by
    # behind_tls_proxy and certificate state rather than a separate file.
    toolshed_nginx_conf: |
      upstream toolshed_backend {
          server 127.0.0.1:{{ toolshed_backend_port }};
      }

      {% macro toolshed_locations() %}
          location /api {
              proxy_set_header Host $host;
              proxy_set_header X-Real-IP $remote_addr;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
              proxy_pass http://toolshed_backend;
          }

          location /auth {
              proxy_set_header Host $host;
              proxy_set_header X-Real-IP $remote_addr;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
              proxy_pass http://toolshed_backend;
          }

          # Django (SignatureAuthentication + per-file friend/owner checks,
          # see files/media_urls.py) decides whether the request is allowed
          # at all; it never streams the bytes itself here (SERVE_X_ACCEL_REDIRECT
          # is on), it just answers with an X-Accel-Redirect to the internal
          # location below, which nginx follows and serves directly from disk.
          location /media {
              proxy_set_header Host $host;
              proxy_set_header X-Real-IP $remote_addr;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
              proxy_pass http://toolshed_backend;
          }

          # Only reachable via the X-Accel-Redirect above, never directly by
          # clients (`internal`) - this is what makes it safe for nginx to
          # serve these bytes itself without reimplementing the access
          # checks Django already did in the /media location.
          location /redirect_media/ {
              internal;
              alias {{ toolshed_data_dir }}/userfiles/;
              # Django would normally set this itself (CORS_ALLOW_ALL_ORIGINS,
              # see settings.py) but never gets to run for a request nginx
              # serves directly - see the comment in files/media_urls.py.
              add_header Access-Control-Allow-Origin * always;
          }

          location /djangoadmin {
              proxy_set_header Host $host;
              proxy_set_header X-Real-IP $remote_addr;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
              proxy_pass http://toolshed_backend;
          }

          location /docs {
              proxy_set_header Host $host;
              proxy_set_header X-Real-IP $remote_addr;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
              proxy_set_header X-Forwarded-Proto {{ toolshed_x_forwarded_proto }};
              proxy_pass http://toolshed_backend;
          }

          location /static/ {
              alias {{ toolshed_static_dir }}/;
              try_files $uri =404;
          }

          location /wiki/ {
              alias {{ toolshed_wiki_dist_dir }}/;
              try_files $uri $uri/ =404;
          }

          location = /wiki {
              return 301 /wiki/;
          }

          # Static fixtures the frontend fetches directly (registration
          # domain list, DoH resolver preference) - see toolshed_register_domains
          # and toolshed_doh_resolvers above.
          location /local/ {
              alias {{ toolshed_local_dir }}/;
              try_files $uri.json =404;
              add_header Content-Type application/json;
          }

          # Vue-router history mode: fall back to index.html for
          # any path that isn't a real static file.
          location / {
              try_files $uri $uri/ /index.html;
          }
      {% endmacro %}

      {% if behind_tls_proxy | default(false) | bool %}
      server {
          listen {{ toolshed_http_port }};
          listen [::]:{{ toolshed_http_port }};
          server_name {{ toolshed_hostnames | join(' ') }};

          client_max_body_size 128M;
          root {{ toolshed_dist_dir }};
          index index.html;
      {{ toolshed_locations() }}
      }
      {% else %}
      {% set tls_active = toolshed_cert.stat.exists | default(false) %}
      server {
          listen {{ toolshed_http_port }};
          listen [::]:{{ toolshed_http_port }};
          server_name {{ toolshed_hostnames | join(' ') }};

          location /.well-known/acme-challenge/ {
              root {{ toolshed_letsencrypt_webroot }};
          }
      {% if tls_active %}

          location / {
              return 301 https://$host$request_uri;
          }
      {% else %}

          client_max_body_size 128M;
          root {{ toolshed_dist_dir }};
          index index.html;
      {{ toolshed_locations() }}
      {% endif %}
      }
      {% if tls_active %}

      server {
          listen 443 ssl;
          listen [::]:443 ssl;
          server_name {{ toolshed_hostnames | join(' ') }};

          ssl_certificate     /etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem;
          ssl_certificate_key /etc/letsencrypt/live/{{ toolshed_domain }}/privkey.pem;

          client_max_body_size 128M;
          root {{ toolshed_dist_dir }};
          index index.html;
      {{ toolshed_locations() }}
      }
      {% endif %}
      {% endif %}

  tasks:
    - name: Install docker.io and nginx
      ansible.builtin.apt:
        name:
          - docker.io
          - nginx
        state: present
        update_cache: true

    - name: Install certbot
      ansible.builtin.apt:
        name: certbot
        state: present
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Ensure docker is running and enabled
      ansible.builtin.systemd:
        name: docker
        state: started
        enabled: true

    - name: Ensure nginx is running and enabled
      ansible.builtin.systemd:
        name: nginx
        state: started
        enabled: true

    - name: Checkout toolshed source
      ansible.builtin.git:
        repo: "{{ toolshed_repo_url | mandatory('toolshed_repo_url must be set as a host_var for ' ~ inventory_hostname) }}"
        dest: "{{ toolshed_src_dir }}"
        version: "{{ toolshed_version | default('stable') }}"
        force: true
        # frontend/extras is registered as a submodule but unused and its
        # pinned commit isn't fetchable from upstream - don't let a broken
        # submodule block the checkout.
        recursive: false

    - name: Create toolshed system user
      ansible.builtin.user:
        name: toolshed
        system: true
        shell: /usr/sbin/nologin
        home: "{{ toolshed_data_dir }}"
        create_home: false
      register: toolshed_user

    - name: Create backend data directories
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        owner: toolshed
        group: toolshed
        mode: "0750"
      loop:
        - "{{ toolshed_data_dir }}"
        - "{{ toolshed_data_dir }}/userfiles"

    # nginx's `location /redirect_media/` (below) reads user-uploaded files
    # straight off disk as www-data - group membership plus the 0750 mode
    # above/FILE_UPLOAD_PERMISSIONS (backend/backend/settings.py) is what
    # makes that readable without loosening it to world-readable.
    - name: Allow nginx to read backend user files
      ansible.builtin.user:
        name: www-data
        groups: toolshed
        append: true
      # New group membership only takes effect for processes started (or
      # forked) after this - nginx's already-running workers won't see it
      # until reloaded.
      notify: reload nginx

    - name: Create frontend static output directory
      ansible.builtin.file:
        path: "{{ toolshed_dist_dir }}"
        state: directory
        owner: www-data
        group: www-data
        mode: "0755"

    - name: Create backend static output directory
      ansible.builtin.file:
        path: "{{ toolshed_static_dir }}"
        state: directory
        owner: www-data
        group: www-data
        mode: "0755"

    - name: Write backend environment file
      ansible.builtin.copy:
        dest: "{{ toolshed_data_dir }}/backend.env"
        # Root-owned and unreadable by the toolshed user on purpose: this is
        # read by the docker daemon (root) via --env-file at container
        # start and injected directly as env vars, so the containerized app
        # - which runs as the toolshed user, see the systemd unit below -
        # never needs filesystem access to its own SECRET_KEY.
        owner: root
        group: root
        mode: "0600"
        content: |
          DEBUG={{ toolshed_debug }}
          SECRET_KEY={{ toolshed_secret_key }}
          ALLOWED_HOSTS={{ toolshed_hostnames | join(',') }}
          SERVE_X_ACCEL_REDIRECT=True
          TOOLSHED_DB_PATH=/data/db.sqlite3
          TOOLSHED_USERFILES_PATH=/data/userfiles
      notify: restart backend

    - name: Build backend docker image
      ansible.builtin.command:
        cmd: >-
          docker build -t {{ toolshed_backend_image }}:{{ toolshed_image_tag }}
          -f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.backend {{ toolshed_src_dir }}/backend
      changed_when: true
      notify: restart backend

    - name: Tag backend image as latest
      ansible.builtin.command:
        cmd: docker tag {{ toolshed_backend_image }}:{{ toolshed_image_tag }} {{ toolshed_backend_image }}:latest
      changed_when: true
      notify: restart backend

    # Dockerfile.backend runs collectstatic at build time, baking the result
    # into the image at /app/staticfiles - copy it out to the host so nginx
    # can serve it directly instead of proxying every asset request to
    # gunicorn. No Django settings/DB access needed, so this can run as a
    # one-off command against the image rather than the container.
    - name: Export backend static files
      ansible.builtin.command:
        cmd: >-
          docker run --rm -v {{ toolshed_static_dir }}:/output
          {{ toolshed_backend_image }}:latest
          sh -c "cp -a /app/staticfiles/. /output/"
      changed_when: true

    - name: Fix ownership of exported backend static files
      ansible.builtin.file:
        path: "{{ toolshed_static_dir }}"
        owner: www-data
        group: www-data
        recurse: true

    - name: Install systemd unit for the backend container
      ansible.builtin.copy:
        dest: /etc/systemd/system/toolshed-backend.service
        owner: root
        group: root
        mode: "0644"
        content: |
          [Unit]
          Description=Toolshed backend (Django) container
          After=docker.service network-online.target
          Requires=docker.service
          Wants=network-online.target

          [Service]
          TimeoutStartSec=0
          Restart=always
          ExecStartPre=-/usr/bin/docker stop {{ toolshed_backend_container }}
          ExecStartPre=-/usr/bin/docker rm {{ toolshed_backend_container }}
          ExecStart=/usr/bin/docker run --rm --name {{ toolshed_backend_container }} \
              --user {{ toolshed_user.uid }}:{{ toolshed_user.group }} \
              --env-file {{ toolshed_data_dir }}/backend.env \
              -v {{ toolshed_data_dir }}:/data \
              -p 127.0.0.1:{{ toolshed_backend_port }}:8000 \
              {{ toolshed_backend_image }}:latest
          ExecStop=/usr/bin/docker stop {{ toolshed_backend_container }}

          [Install]
          WantedBy=multi-user.target
      notify: restart backend

    # Installed before the bootstrap nginx flush_handlers below (which
    # flushes every pending handler, not just reload nginx) - otherwise a
    # fresh host would flush "restart backend" before this unit file exists
    # and fail with "Could not find the requested service".
    - name: Ensure toolshed-backend service is enabled and started
      ansible.builtin.systemd:
        name: toolshed-backend
        daemon_reload: true
        enabled: true
        state: started

    - name: Build frontend builder docker image
      ansible.builtin.command:
        cmd: >-
          docker build -t {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
          -f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.frontend {{ toolshed_src_dir }}/frontend
      changed_when: true

    - name: Run frontend builder once to export the static build
      ansible.builtin.command:
        cmd: docker run --rm -v {{ toolshed_dist_dir }}:/output {{ toolshed_frontend_image }}:{{ toolshed_image_tag }}
      changed_when: true

    - name: Fix ownership of exported frontend build
      ansible.builtin.file:
        path: "{{ toolshed_dist_dir }}"
        owner: www-data
        group: www-data
        recurse: true

    - name: Create wiki static output directory
      ansible.builtin.file:
        path: "{{ toolshed_wiki_dist_dir }}"
        state: directory
        owner: www-data
        group: www-data
        mode: "0755"

    - name: Build wiki builder docker image
      ansible.builtin.command:
        cmd: >-
          docker build -t {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
          -f {{ toolshed_src_dir }}/deploy/prod/Dockerfile.wiki {{ toolshed_src_dir }}
      changed_when: true

    - name: Run wiki builder once to export the static site
      ansible.builtin.command:
        cmd: docker run --rm -v {{ toolshed_wiki_dist_dir }}:/output {{ toolshed_wiki_image }}:{{ toolshed_image_tag }}
      changed_when: true

    - name: Fix ownership of exported wiki build
      ansible.builtin.file:
        path: "{{ toolshed_wiki_dist_dir }}"
        owner: www-data
        group: www-data
        recurse: true

    - name: Create local fixtures directory
      ansible.builtin.file:
        path: "{{ toolshed_local_dir }}"
        state: directory
        owner: www-data
        group: www-data
        mode: "0755"

    - name: Write registration domain list fixture
      ansible.builtin.copy:
        dest: "{{ toolshed_local_dir }}/domains.json"
        owner: www-data
        group: www-data
        mode: "0644"
        content: "{{ toolshed_register_domains | to_nice_json }}"

    - name: Write DoH resolver fixture
      ansible.builtin.copy:
        dest: "{{ toolshed_local_dir }}/dns.json"
        owner: www-data
        group: www-data
        mode: "0644"
        content: "{{ toolshed_doh_resolvers | to_nice_json }}"

    - name: Create ACME HTTP-01 challenge webroot
      ansible.builtin.file:
        path: "{{ toolshed_letsencrypt_webroot }}"
        state: directory
        owner: www-data
        group: www-data
        mode: "0755"
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Check for an existing Let's Encrypt certificate
      ansible.builtin.stat:
        path: "/etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem"
      register: toolshed_cert
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Configure nginx site for toolshed (bootstrap)
      ansible.builtin.copy:
        dest: /etc/nginx/sites-available/toolshed.conf
        owner: root
        group: root
        mode: "0644"
        content: "{{ toolshed_nginx_conf }}"
      notify: reload nginx

    - name: Remove default nginx site
      ansible.builtin.file:
        path: /etc/nginx/sites-enabled/default
        state: absent
      notify: reload nginx

    - name: Enable toolshed nginx site
      ansible.builtin.file:
        src: /etc/nginx/sites-available/toolshed.conf
        dest: /etc/nginx/sites-enabled/toolshed.conf
        state: link
      notify: reload nginx

    # Certbot's webroot check (below) needs nginx already serving the
    # bootstrap config from the tasks above, so force the reload now
    # instead of waiting for the end of the play.
    - name: Apply the bootstrap nginx config now
      ansible.builtin.meta: flush_handlers

    - name: Ensure the certbot renewal deploy-hook directory exists
      ansible.builtin.file:
        path: /etc/letsencrypt/renewal-hooks/deploy
        state: directory
        mode: "0755"
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Reload nginx after certbot renews a certificate
      ansible.builtin.copy:
        dest: /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh
        owner: root
        group: root
        mode: "0755"
        content: |
          #!/bin/sh
          systemctl reload nginx
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Obtain or renew the Let's Encrypt certificate
      ansible.builtin.command:
        cmd: >-
          certbot certonly --webroot -w {{ toolshed_letsencrypt_webroot }}
          -d {{ toolshed_hostnames | join(' -d ') }}
          --non-interactive --agree-tos
          -m {{ toolshed_letsencrypt_email | mandatory('toolshed_letsencrypt_email must be set as a host_var for ' ~ inventory_hostname ~ ' since behind_tls_proxy is false there') }}
      register: toolshed_certbot
      changed_when: "'Certificate not yet due for renewal' not in toolshed_certbot.stdout"
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Re-check the certificate now that certbot has run
      ansible.builtin.stat:
        path: "/etc/letsencrypt/live/{{ toolshed_domain }}/fullchain.pem"
      register: toolshed_cert
      when: not (behind_tls_proxy | default(false) | bool)

    - name: Configure nginx site for toolshed (final)
      ansible.builtin.copy:
        dest: /etc/nginx/sites-available/toolshed.conf
        owner: root
        group: root
        mode: "0644"
        content: "{{ toolshed_nginx_conf }}"
      notify: reload nginx

  handlers:
    - name: validate nginx config
      ansible.builtin.command: nginx -t
      listen: reload nginx
      changed_when: false

    - name: reload nginx
      ansible.builtin.systemd:
        name: nginx
        state: reloaded
      listen: reload nginx

    - name: restart backend
      ansible.builtin.systemd:
        name: toolshed-backend
        daemon_reload: true
        state: restarted
      listen: restart backend