feat(hetzner): add netdata

- Update virtualcam website and API to latest repository revisions.
- Add Google OAuth client ID and secrets decryption for Google auth.
- Enable Shkeeper BTC/USD payment processing (`BYPASS_PAYMENTS=false`) and add service dependencies.
- Update Caddy `admin_gate` IP access rules and remove redundant `admin_gate` import from app proxy.
- Configure weekly automatic Nix garbage collection (`--delete-older-than 14d`) and nix store optimization.

.gitea/workflows/ci.yml

@@ -13,10 +13,13 @@
 #     PATH before invoking `nix`. This uses the host's actual Lix rather than installing a
 #     second Nix client that would mismatch the running Lix daemon.
 #   * Gitea context vars (gitea.workflow / gitea.head_ref / gitea.sha) for concurrency.
-#   * matrix over x86_64-linux + aarch64-linux for flake-check (--no-build, eval-only).
-#     Pre-commit checks run on x86_64-linux only: building aarch64 derivations needs
-#     QEMU binfmt (registered via boot.binfmt.emulatedSystems) + nix extra-platforms,
-#     which require a nixos-rebuild switch that hasn't been applied yet.
+#   * matrix over x86_64-linux + aarch64-linux.
+#     All jobs use --no-build (eval-only) because the runner has 2 cores / 4GB RAM:
+#     building nix derivations would exhaust memory and crash the server.
+#     Pre-commit hooks run locally via git-hooks-nix on the developer's machine.
+#   * aarch64 builds need QEMU binfmt (boot.binfmt.emulatedSystems) + nix extra-platforms,
+#     which require a nixos-rebuild switch not yet applied. Flake-check for aarch64
+#     runs eval-only (no build) and passes.
 
 name: CI
 
@@ -61,24 +64,3 @@ jobs:
         run: |
           export PATH=/run/current-system/sw/bin:$PATH
           nix flake check --no-build --system ${{ matrix.system }}
-
-  pre-commit:
-    name: Pre-commit checks (${{ matrix.system }})
-    runs-on: native
-    strategy:
-      fail-fast: false
-      matrix:
-        # aarch64-linux omitted: building aarch64 derivations needs QEMU binfmt +
-        # extra-platforms; system hasn't been rebuilt to apply them yet.
-        system:
-          - x86_64-linux
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-        with:
-          persist-credentials: false
-
-      - name: Pre-commit checks (${{ matrix.system }})
-        run: |
-          export PATH=/run/current-system/sw/bin:$PATH
-          nix build .#checks.${{ matrix.system }}.pre-commit

result

@@ -0,0 +1 @@
+/nix/store/p2h0fr7k47yrx3x0qkr3rwsp5nf5bj30-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951

servers/hetzner/hosts/hetzner/default.nix

@@ -12,14 +12,15 @@
     ../../modules/services/wireguard.nix
     ../../modules/services/fail2ban.nix
     ../../modules/services/vaultwarden.nix
-    ../../modules/services/mailserver.nix
-    ../../modules/services/snappymail.nix
     ../../modules/services/rustdesk.nix
     ../../modules/services/wrxproxy.nix
     ../../modules/services/watchtower.nix
     ../../modules/services/tlsa-updater.nix
     ../../modules/services/shkeeper.nix
     ../../modules/services/backup.nix
+    ../../modules/services/stalwart.nix
+    ../../modules/services/virtualcam.nix
+    ../../modules/services/netdata.nix
   ];
 
   # Only 4GB RAM — limit nix builds to one core at a time to avoid OOM

servers/hetzner/hosts/hetzner/dns-zone.nix

@@ -2,7 +2,7 @@
 # This is the authoritative DNS zone served by CoreDNS
 # Update serial number on changes
 {
-  serial = "2026071003";
+  serial = "2026071004";
   adminEmail = "abuse.severijnse.eu";
   nameservers = ["ns1.severijnse.eu" "ns2.severijnse.eu"];
   ipv4 = "49.13.92.205";

servers/hetzner/hosts/hetzner/hardware.nix

@@ -17,7 +17,7 @@
   swapDevices = [
     {
       device = "/swap";
-      size = 8192;
+      size = 4096;
     }
   ];
 

servers/hetzner/modules/services/backup.nix

@@ -1,41 +1,93 @@
-{pkgs, ...}: let
-  backupScript = pkgs.writeShellScript "weekly-backup" ''
-    BACKUP_DIR="/home/admin/backups"
-    SRC="/home/admin"
-    DATE=$(date +%Y-%m-%dT%H-%M-%S)
-    FILENAME="weekly-backup-$DATE.tar.gz"
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  # sops-encrypted secrets (single file holds all service secrets).
+  secretsFile = ../../secrets/secrets.yaml;
+  # Root-only runtime files restic reads from (0600 root).
+  runtimeDir = "/var/lib/restic";
+  passwordFile = "/var/lib/restic/.password";
+  environmentFile = "/var/lib/restic/environment";
+  # Backblaze B2 backend, per restic docs: b2:bucketname.
+  repo = "b2:hetzner-severijnse";
 
-    mkdir -p "$BACKUP_DIR"
-    # Backup everything under /home/admin EXCEPT:
-    # - The backups dir itself (infinite loop)
-    # - DMS mail data (GBs of email, backed up separately)
-    # - NixOS-managed service data (at their own paths below)
-    tar czf "$BACKUP_DIR/$FILENAME" \
-      --exclude="$BACKUP_DIR" \
-      --exclude="/home/admin/backups" \
-      --exclude="/home/admin/dms/mail-data" \
-      --exclude="/home/admin/dms/mail-state" \
-      "$SRC"
+  # Materialize the restic password and B2 credentials from sops into
+  # root-only files, so secrets are never world-readable in the Nix store.
+  writeSecrets = pkgs.writeShellScript "restic-write-secrets" ''
+    set -euo pipefail
+    mkdir -p ${runtimeDir}
+    ${pkgs.sops}/bin/sops \
+      --decrypt --extract '["restic_password"]' \
+      --input-type yaml --output-type yaml ${secretsFile} \
+      | tr -d '\n' > "${passwordFile}"
+    chmod 0600 "${passwordFile}"
 
-    # Prune backups older than 14 days
-    find "$BACKUP_DIR" -name "weekly-backup-*" -mtime +14 -delete
+    : > "${environmentFile}"
+    chmod 0600 "${environmentFile}"
+    ${pkgs.sops}/bin/sops --decrypt --input-type yaml --output-type yaml ${secretsFile} |
+      ${pkgs.gnused}/bin/sed -nE \
+        's/^b2_key_id: (.*)/B2_ACCOUNT_ID=\1/p; s/^b2_application_key: (.*)/B2_ACCOUNT_KEY=\1/p' \
+        >> "${environmentFile}"
   '';
 in {
-  systemd.services.weekly-backup = {
-    description = "Weekly backup of home directory";
-    path = with pkgs; [coreutils gnutar findutils];
+  systemd.services.restic-password = {
+    description = "Materialize restic repository password and B2 credentials from sops";
+    wantedBy = ["multi-user.target"];
+    # The age key lives in /etc/age/keys.txt; the service must know where it is
+    # and needs a HOME for age to report its user config directory.
+    environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
     serviceConfig = {
       Type = "oneshot";
-      ExecStart = "${backupScript}";
-      User = "root";
+      Environment = ["HOME=/root"];
+      ExecStart = "${writeSecrets}";
     };
   };
 
-  systemd.timers.weekly-backup = {
-    wantedBy = ["timers.target"];
+  # B2 credentials are supplied via environmentFile (B2_ACCOUNT_ID / B2_ACCOUNT_KEY),
+  # matching the official module example which combines `repository` and `environmentFile`.
+  services.restic.backups.localbackup = {
+    repository = "b2:hetzner-severijnse";
+    environmentFile = environmentFile;
+    passwordFile = passwordFile;
+    initialize = true;
+    paths = [
+      "/home/admin"
+      "/var/lib/postgresql"
+      "/var/lib/gitea"
+      "/var/lib/caddy"
+      "/var/lib/virtualcam"
+      "/var/lib/stalwart"
+      "/var/lib/bulwark"
+      "/var/lib/coredns"
+      "/etc/nixos"
+    ];
+    exclude = [
+      "/home/admin/backups"
+      "/home/admin/.opencode"
+      "/home/admin/.local"
+      "/home/admin/.npm"
+      "/home/admin/.config"
+      "*.log"
+      "*.log.*"
+      "**/.cache"
+    ];
     timerConfig = {
       OnCalendar = "Mon *-*-* 03:00:00";
       Persistent = true;
+      RandomizedDelaySec = "15m";
     };
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 6"
+    ];
+    runCheck = true;
+  };
+
+  # The backup must never run before the secrets exist.
+  systemd.services."restic-backups-localbackup" = {
+    requires = ["restic-password.service"];
+    after = ["restic-password.service"];
   };
 }

servers/hetzner/modules/services/caddy.nix

@@ -1,4 +1,4 @@
-{...}: let
+{unstablePkgs, ...}: let
   antiScrape = ''
     @bad_bot {
       header_regexp User-Agent "(?i)(scrapy|cpython-requests|python-requests|curl|wget|go-http-client|ltx71|petalbot|bytespider|dotbot|ahrefsbot|semrushbot|mj12bot|dataforseo|facebookexternalhit|claudebot|anthropic-ai|perplexity|gptbot|chatgpt-user|omnisci|imgproxy|ccbot|exabot|360spider|baiduspider|sogou|duckduckgo|amazonbot|cohere-ai|diffbot|imagesiftbot).*"
@@ -12,6 +12,7 @@ in {
   services.caddy = {
     enable = true;
     group = "caddy";
+    package = unstablePkgs.caddy;
     dataDir = "/var/lib/caddy";
     logDir = "/var/log/caddy";
     globalConfig = ''
@@ -23,9 +24,16 @@ in {
     # Global Caddyfile snippets (shared across all virtual hosts).
     extraConfig = ''
       (admin_gate) {
-        @notvpn not remote_ip 10.8.0.0/24
+        @notvpn not remote_ip 127.0.0.1 ::1 10.8.0.0/24 fd10:8::/64 49.13.92.205 2a01:4f8:c014:2585::1
         respond @notvpn "Forbidden" 403
       }
+      # Restrict access to the server itself (loopback + public IPs) or the
+      # WireGuard VPN. Gitea reaches this from the server when publishing a
+      # release, and VPN clients can download from it; the public cannot.
+      (release_gate) {
+        @deny not remote_ip 127.0.0.1 ::1 10.8.0.0/24 fd10:8::/64 49.13.92.205 2a01:4f8:c014:2585::1
+        respond @deny "Forbidden" 403
+      }
       # Security headers applied to every response of every site that
       # imports this snippet (covers all current and future hosts).
       # For proxied hosts whose upstream sets its own copies, strip them
@@ -133,7 +141,6 @@ in {
           import security_headers
           import csp
           ${antiScrape}
-          import admin_gate
           header Strict-Transport-Security "max-age=31536000;"
           reverse_proxy 127.0.0.1:1001
           encode zstd gzip
@@ -154,13 +161,33 @@ in {
         extraConfig = ''
           import security_headers
           ${antiScrape}
-          reverse_proxy 127.0.0.1:8888 {
-            # Strip copies set by the upstream SnappyMail container so we
-            # emit exactly one correct value of each security header.
-            header_down -X-Frame-Options
-            header_down -X-XSS-Protection
-            header_down -X-Content-Type-Options
+          # Stalwart JMAP API (http-management listener on 8080): required so the
+          # browser can reach /.well-known/jmap, /jmap/session and the JMAP
+          # upload/download/eventsource/ws paths at the public origin.
+          handle /.well-known/jmap {
+            reverse_proxy 127.0.0.1:8080
           }
+          # NOTE: use handle (not handle_path) so the /jmap prefix is preserved.
+          handle /jmap {
+            reverse_proxy 127.0.0.1:8080
+          }
+          handle /jmap/* {
+            reverse_proxy 127.0.0.1:8080
+          }
+          handle {
+            # Bulwark webmail (JMAP client for Stalwart), running on host port 3002.
+            reverse_proxy 127.0.0.1:3002
+          }
+          encode zstd gzip
+        '';
+      };
+
+      "admin.mail.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          ${antiScrape}
+          # Stalwart webadmin UI (served by the http-management listener on 8080).
+          reverse_proxy 127.0.0.1:8080
           encode zstd gzip
         '';
       };
@@ -218,12 +245,48 @@ in {
           import security_headers
           import csp
           ${antiScrape}
-          import admin_gate
           reverse_proxy 127.0.0.1:5000
           encode zstd gzip
         '';
       };
 
+      "virtualcam.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          import csp
+          ${antiScrape}
+          basic_auth {
+            chan $2a$14$7ZeNpGT0L68uZwzdWBcF0OulzhrYbfAs232Ojt//LHQ1qvXU4x32O
+          }
+          reverse_proxy 127.0.0.1:3001
+          encode zstd gzip
+        '';
+      };
+
+      "api.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          # Machine-facing license API for the Android app: no antiScrape
+          # bot-blocking (it would drop legitimate non-browser clients) and no
+          # admin_gate (endpoints are protected by the bearer token).
+          reverse_proxy 127.0.0.1:3004
+          encode zstd gzip
+        '';
+      };
+
+      "releases.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          root * /srv/releases
+          file_server browse
+          encode zstd gzip
+
+          handle /private* {
+            import release_gate
+          }
+        '';
+      };
+
       "http://ip.severijnse.eu" = {
         extraConfig = ''
           import security_headers
@@ -238,5 +301,6 @@ in {
   systemd.tmpfiles.rules = [
     "d /srv 0755 caddy caddy -"
     "d /srv/jory 0755 caddy caddy -"
+    "d /srv/releases 0755 admin caddy -"
   ];
 }

servers/hetzner/modules/services/coredns.nix

@@ -2,7 +2,7 @@
   zoneFile = pkgs.writeText "severijnse.eu.db" ''
     $ORIGIN severijnse.eu.
     $TTL 3600
-    severijnse.eu. 3600 IN SOA ns1.severijnse.eu. abuse.severijnse.eu. 2026071003 3600 1800 1209600 86400
+    severijnse.eu. 3600 IN SOA ns1.severijnse.eu. abuse.severijnse.eu. 2026071004 3600 1800 1209600 86400
          IN    NS     ns1.severijnse.eu.
          IN    NS     ns2.severijnse.eu.
 
@@ -11,12 +11,14 @@
     ns1  IN    A      49.13.92.205
     ns2  IN    A      49.13.92.205
     mail IN    A      49.13.92.205
+    admin.mail IN    A      49.13.92.205
 
     @    IN    AAAA   2a01:4f8:c014:2585::1
     www  IN    AAAA   2a01:4f8:c014:2585::1
     ns1  IN    AAAA   2a01:4f8:c014:2585::1
     ns2  IN    AAAA   2a01:4f8:c014:2585::1
     mail IN    AAAA   2a01:4f8:c014:2585::1
+    admin.mail IN    AAAA   2a01:4f8:c014:2585::1
 
     *.severijnse.eu.  IN A      49.13.92.205
     *.severijnse.eu.  IN AAAA   2a01:4f8:c014:2585::1
@@ -25,7 +27,7 @@
 
     mail._domainkey.severijnse.eu. 300 IN TXT (
       "v=DKIM1; k=rsa; "
-      "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8/"
+      "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8K/"
       "JaoTC8OURk365+/aBSmWXUCCBphCx43QWfC9h8GMQ6PUIqawkz5CcGPT7X7hPdwHQcd5Vn3CqmABptxdwshdkBjZs"
       "oi79BOo9ZrQSTY7iiLcOP7hVVC9Ad+ydlZ4MWGfy5BxgyTGrrtuSuLcM219oqdovIvr2EtXs8AMx5fyXplKE3R/"
       "YlwF2Jcy50Gmb5y/E9pOaFjVv8HXUmKsvuhA2b8K+rt0WVHNc3dvbgZUl8bGAQIDAQAB"

servers/hetzner/modules/services/fail2ban.nix

@@ -4,7 +4,6 @@
     image = "crazymax/fail2ban:latest";
     autoStart = true;
     volumes = [
-      "/home/admin/dms/mail-logs:/var/log/mail:ro"
       "/home/admin/fail2ban/data:/data:Z"
     ];
     environment = {
@@ -15,8 +14,4 @@
       "--cap-add=NET_ADMIN"
     ];
   };
-
-  systemd.tmpfiles.rules = [
-    "d /var/log/mail 0755 root root -"
-  ];
 }

servers/hetzner/modules/services/netdata.nix

@@ -0,0 +1,32 @@
+{
+  config,
+  pkgs,
+  ...
+}: {
+  #
+  # Netdata
+  #
+  services.netdata = {
+    enable = true;
+    config = {
+      global = {
+        "memory mode" = "ram";
+        "debug log" = "none";
+        "access log" = "none";
+        "error log" = "syslog";
+      };
+      web = {
+        "bind to" = "127.0.0.1:19999";
+      };
+    };
+  };
+
+  services.caddy.virtualHosts."netdata.severijnse.eu" = {
+    extraConfig = ''
+      import security_headers
+      import csp
+      reverse_proxy 127.0.0.1:19999
+      encode zstd gzip
+    '';
+  };
+}

servers/hetzner/modules/services/snappymail.nix

@@ -1,23 +0,0 @@
-{...}: {
-  virtualisation.oci-containers.containers.snappymail = {
-    image = "djmaze/snappymail:latest";
-    autoStart = true;
-    ports = ["127.0.0.1:8888:8888"];
-    volumes = [
-      "/home/admin/snappymail-data:/var/lib/snappymail:Z"
-    ];
-    environment = {
-      TZ = "Europe/Berlin";
-    };
-    extraOptions = [
-      "--label=com.centurylinklabs.watchtower.enable=true"
-    ];
-  };
-
-  # Ensure the persistent data dir exists so podman's :Z relabel (statfs) succeeds on first boot.
-  # Owned by 82:82 (www-data) because the container's PHP worker runs as UID 82 and must be
-  # able to write to /var/lib/snappymail (SnappyMail checks is_writable on that path).
-  systemd.tmpfiles.rules = [
-    "d /home/admin/snappymail-data 0755 82 82 - -"
-  ];
-}

servers/hetzner/modules/services/stalwart.nix

@@ -0,0 +1,310 @@
+{
+  lib,
+  pkgs,
+  unstablePkgs,
+  ...
+}: let
+  # Caddy's dist dir (see tlsa-updater.nix): cert 0644, key 0640 root:root.
+  # The Stalwart service runs as "stalwart"; grant it read access to the key.
+  certDir = "/var/lib/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
+  # sops-encrypted secrets (single file holds all service secrets), same as backup.nix.
+  secretsFile = ../../secrets/secrets.yaml;
+  # Root-only runtime file holding the fallback-admin password hash (0600 root).
+  adminHashFile = "/var/lib/stalwart/.admin-hash";
+
+  # Materialize the fallback-admin password hash from sops into a root-only file.
+  writeAdminHash = pkgs.writeShellScript "stalwart-write-admin-hash" ''
+    set -euo pipefail
+    install -d -o root -g root -m 0755 "$(dirname ${adminHashFile})"
+    ${pkgs.sops}/bin/sops \
+      --decrypt --extract '["stalwart_admin_hash"]' \
+      --input-type yaml --output-type yaml ${secretsFile} \
+      | tr -d '\n' > "${adminHashFile}"
+    chmod 0600 "${adminHashFile}"
+  '';
+
+  # The outbound DKIM key is the existing opendkim key from docker-mailserver
+  # (selector "mail", domain severijnse.eu, PKCS#8 RSA 2048). Install it into the
+  # stalwart-owned data dir so the server can read it and it is covered by the
+  # /var/lib/stalwart backup.
+  dkimKeyDir = "/var/lib/stalwart/dkim/severijnse.eu";
+  dkimKeySrc = "/home/admin/dms/config/opendkim/keys/severijnse.eu/mail.private";
+  writeDkimKey = pkgs.writeShellScript "stalwart-write-dkim-key" ''
+    set -euo pipefail
+    install -d -o stalwart -g stalwart -m 0750 ${dkimKeyDir}
+    install -o stalwart -g stalwart -m 0640 ${dkimKeySrc} ${dkimKeyDir}/mail.private
+  '';
+in {
+  # The hetzner host is built with nixos-24.05, which ships its own
+  # `services.stalwart-mail` module (for the old 0.8.x package). We want the
+  # 0.15.5 module from the locked nixpkgs-unstable instead, so we must exclude
+  # the 24.05 default module (which defines the same option namespace) to avoid
+  # the rename-based infinite recursion, and import the unstable one in its place.
+  disabledModules = [
+    "services/mail/stalwart-mail.nix"
+  ];
+
+  imports = [
+    "${unstablePkgs.path}/nixos/modules/services/mail/stalwart.nix"
+  ];
+
+  services.stalwart = {
+    enable = true;
+    stateVersion = "26.05";
+    package = unstablePkgs.stalwart;
+
+    # Public listeners. The host firewall (networking.nix) already allows
+    # 25/143/465/587/993, so openFirewall stays off: the module would also open
+    # every other listener port incl. the 8080 webadmin.
+    openFirewall = false;
+
+    settings = {
+      # EHLO / hostname for the server (docs server.hostname).
+      server.hostname = "mail.severijnse.eu";
+
+      # Public origin the JMAP/webadmin API is served from, advertised in the
+      # JMAP session (docs server/core/network.md "http.url"). Value must be a
+      # JScript expression, hence the single-quoted string literal. Without this
+      # Stalwart advertises http://mail.severijnse.eu:8080, which the browser
+      # refuses to fetch and breaks Bulwark webmail.
+      http.url = "'https://mail.severijnse.eu'";
+
+      certificate."mail-severijnse-eu" = {
+        cert = "%{file:${certDir}/mail.severijnse.eu.crt}%";
+        private-key = "%{file:${certDir}/mail.severijnse.eu.key}%";
+        # Docs server/tls/certificates.md: used when the client sends no SNI.
+        default = true;
+      };
+
+      server.tls = {
+        certificate = "mail-severijnse-eu";
+        enable = true;
+        implicit = false;
+      };
+
+      # Public listeners (docs server/listener.md + protocol, tls.implicit override).
+      # Bind "[::]:port" for dual-stack IPv4+IPv6 (docs: "to bind a listener to
+      # all interfaces"); listing both 0.0.0.0 and [::] makes the [::] bind fail
+      # with EADDRINUSE on kernels with net.ipv6.bindv6only=0.
+      # 143/587 use STARTTLS (server.tls.implicit=false default), 993/465 the
+      # implicit-TLS variants, 25 the plain (STARTTLS) MX port.
+      server.listener = {
+        "imap" = {
+          bind = ["[::]:143"];
+          protocol = "imap";
+        };
+        "imaps" = {
+          bind = ["[::]:993"];
+          protocol = "imap";
+          tls.implicit = true;
+        };
+        "smtp" = {
+          bind = ["[::]:25"];
+          protocol = "smtp";
+        };
+        "smtp-submission" = {
+          bind = ["[::]:587"];
+          protocol = "smtp";
+        };
+        "smtp-submissions" = {
+          bind = ["[::]:465"];
+          protocol = "smtp";
+          tls.implicit = true;
+        };
+        "http-management" = {
+          bind = ["127.0.0.1:8080"];
+          protocol = "http";
+        };
+      };
+
+      # Auth per docs mta/inbound/auth.md (AUTH stage): authentication is
+      # disabled on the plain SMTP listener (port 25) and required everywhere
+      # else (IMAP + submission). Only offer PLAIN/LOGIN over TLS, so clear-text
+      # listeners (143/587 pre-STARTTLS) advertise no SASL mechanisms. This
+      # mirrors the code defaults (crates/common/src/config/smtp/session.rs).
+      # The JMAP/webadmin "http-management" listener (bind 127.0.0.1:8080) is
+      # plain HTTP and reachable only from localhost, where bulwark connects;
+      # its basic-auth uses the "plain" mechanism, so it must be exempted from
+      # the is_tls gate or webmail logins fail with "Authentication not allowed".
+      session.auth.mechanisms = [
+        {
+          "if" = "local_port != 25 && (is_tls || listener == 'http-management')";
+          "then" = "[plain, login]";
+        }
+        {"else" = false;}
+      ];
+      session.auth.directory = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = "'internal'";
+        }
+        {"else" = false;}
+      ];
+      session.auth.require = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = true;
+        }
+        {"else" = false;}
+      ];
+
+      # Outbound DKIM signing (docs mta/authentication/dkim/sign): sign with the
+      # "mail" signature on everything submitted via non-25 listeners; do not sign
+      # inbound mail received on the plain "smtp" listener.
+      auth.dkim.sign = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = "['mail']";
+        }
+        {"else" = false;}
+      ];
+
+      # ARC sealing uses the same "mail" signature (docs mta/authentication/arc).
+      # The code default ('rsa-' + report.domain) would reference a signature
+      # name that does not exist and log "ARC sealer not found".
+      auth.arc.seal = "'mail'";
+
+      # Every *downstream* sign rule defaults to signing with
+      # ['rsa-<report.domain>', 'ed25519-<report.domain>'] (queue.rs/report.rs),
+      # names that do not exist here and log "DKIM signer not found" on DSNs and
+      # reports. Point them all at the real "mail" signature.
+      report.dsn.sign = "['mail']";
+      report.spf.sign = "['mail']";
+      report.dmarc.sign = "['mail']";
+      report.dmarc.aggregate.sign = "['mail']";
+      report.tls.aggregate.sign = "['mail']";
+
+      # Reuse the existing opendkim key (selector mail) so no DNS change is needed.
+      signature.mail = {
+        private-key = "%{file:/var/lib/stalwart/dkim/severijnse.eu/mail.private}%";
+        domain = "severijnse.eu";
+        selector = "mail";
+        headers = ["From" "To" "Date" "Subject" "Message-Id"];
+        algorithm = "rsa-sha256";
+        canonicalization = "relaxed/relaxed";
+        set-body-length = false;
+      };
+
+      # Fallback admin (auth/authorization/administrator.md): bootstrap admin with
+      # every permission, used to create the internal-directory accounts via the
+      # management REST API / CLI. Secret is a SHA-512-crypt hash, injected via
+      # LoadCredential (services.stalwart.credentials) so no secret lands in the
+      # Nix store.
+      authentication."fallback-admin" = {
+        user = "admin";
+        secret = "%{file:/run/credentials/stalwart.service/stalwart-admin}%";
+      };
+
+      # Route docs mta/outbound/routing.md + strategy.md: local domains → local
+      # store, everything else → MX. The strategy names are defined explicitly
+      # (docs define queue.route.mx/local; the built-in fallback in
+      # core.rs:get_route_or_default only kicks in for undeclared names).
+      queue.strategy.route = [
+        {
+          "if" = "is_local_domain('', rcpt_domain)";
+          "then" = "'local'";
+        }
+        {"else" = "'mx'";}
+      ];
+      queue.route."mx" = {
+        type = "mx";
+        ip-lookup = "ipv4_then_ipv6";
+      };
+      queue.route."local" = {
+        type = "local";
+      };
+    };
+  };
+
+  # The module's service runs as user/group "stalwart" (ProtectHome=true,
+  # ProtectSystem=strict). The TLS key tlsa-updater installs is 0640 root:root;
+  # regrant it to the stalwart group after every cert sync so stalwart can serve TLS.
+  systemd.services.stalwart = {
+    after = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"];
+    requires = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"];
+  };
+
+  # Make the management CLI available for account creation and maildir import
+  # (docs management/cli/). Version-pinned to the locked unstable nixpkgs.
+  environment.systemPackages = [unstablePkgs.stalwart-cli];
+
+  # Materialize the fallback-admin password hash from sops before stalwart starts.
+  # The admin hash is a SHA-512-crypt value, safe to pass through a root-only file.
+  systemd.services.stalwart-admin-secret = {
+    description = "Materialize Stalwart fallback-admin password hash from sops";
+    wantedBy = ["multi-user.target"];
+    before = ["stalwart.service"];
+    # The age key lives in /etc/age/keys.txt; the service must know where it is
+    # and needs a HOME for age to report its user config directory.
+    environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
+    serviceConfig = {
+      Type = "oneshot";
+      Environment = ["HOME=/root"];
+      ExecStart = "${writeAdminHash}";
+    };
+  };
+
+  # Install the outbound DKIM key into the stalwart data dir before the service starts.
+  systemd.services.stalwart-dkim = {
+    description = "Install Stalwart outbound DKIM key";
+    wantedBy = ["multi-user.target"];
+    before = ["stalwart.service"];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${writeDkimKey}";
+    };
+  };
+
+  # LoadCredential: expose the materialized hash to stalwart only at
+  # /run/credentials/stalwart.service/stalwart-admin (see `credentials` option
+  # in the upstream module; the value is the source path on disk).
+  services.stalwart.credentials.stalwart-admin = adminHashFile;
+
+  systemd.services.stalwart-cert-perm = {
+    description = "Grant stalwart read access to its TLS private key";
+    # Belt-and-suspenders: tlsa-update already chgrps the key after every sync;
+    # this guarantees the group grant also exists at first boot, before stalwart
+    # starts (previously ordered after stalwart, so a fresh sync could leave a
+    # root:root key and webadmin reload would fail with EACCES).
+    after = ["tlsa-update.service"];
+    before = ["stalwart.service"];
+    partOf = ["tlsa-update.service"];
+    wantedBy = ["multi-user.target"];
+    path = [pkgs.coreutils];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${pkgs.coreutils}/bin/chgrp stalwart ${certDir}/mail.severijnse.eu.key";
+      ExecStartPost = "${pkgs.coreutils}/bin/chmod 0640 ${certDir}/mail.severijnse.eu.key";
+    };
+  };
+
+  # Bulwark webmail (self-hosted JMAP webmail for Stalwart). Serving on
+  # mail.severijnse.eu behind Caddy (see caddy.nix). It connects to Stalwart's
+  # JMAP endpoint at 127.0.0.1:8080, so uses host networking. Next.js defaults
+  # to POST_SIZE/etc via env; JMAP_SERVER_URL points at the Stalwart http
+  # listener which serves JMAP at /jmap.
+  virtualisation.oci-containers.containers.bulwark = {
+    image = "ghcr.io/bulwarkmail/webmail:latest";
+    autoStart = true;
+    volumes = [
+      "/var/lib/bulwark:/app/data:Z"
+    ];
+    environment = {
+      # JMAP_SERVER_URL is the public origin (the browser uses it verbatim for
+      # /.well-known/jmap + the session apiUrl). HOSTNAME stays 127.0.0.1 so
+      # Next.js binds to loopback (caddy reverse_proxy's 127.0.0.1:3002); binding
+      # to the public hostname made the container unreachable for caddy (502).
+      JMAP_SERVER_URL = "https://mail.severijnse.eu";
+      HOSTNAME = "127.0.0.1";
+      PORT = "3002";
+    };
+    extraOptions = [
+      "--network=host"
+      "--label=com.centurylinklabs.watchtower.enable=true"
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/bulwark 0755 1001 1001 - -"
+  ];
+}

servers/hetzner/modules/services/tlsa-updater.nix

@@ -1,11 +1,15 @@
 {pkgs, ...}: let
   # Caddy's canonical certificate storage (XDG data dir). Renewals land here,
-  # owned caddy:caddy 0600 — the mail server's non-root Postfix/Dovecot cannot
-  # read it directly, so we copy it into a world-readable distribution dir.
+  # owned caddy:caddy 0600 — stalwart cannot read it directly, so we copy it
+  # into a distribution dir stalwart's service (user "stalwart") can reach.
   caddyCertDir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
-  # World-readable distribution dir mounted (RO) into the mail server container.
+  # Distribution dir read by stalwart (cert 0644, private key regranted to the
+  # "stalwart" group by systemd.services.stalwart-cert-perm).
   distCertDir = "/var/lib/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
   zoneFile = "/var/lib/coredns/zones/severijnse.eu.db";
+  # Records the SPKI hash applied at the last restart of stalwart, so cert
+  # renewals trigger exactly one restart and unchanged certs never do.
+  stateFile = "/var/lib/tlsa-update/.last-spki";
 
   syncScript = pkgs.writeShellScript "tlsa-update" ''
     set -euo pipefail
@@ -22,13 +26,17 @@
       exit 0
     fi
 
-    # 1) Propagate Caddy's renewed certificate into the distribution dir the
-    #    mail server mounts. Caddy stores certs 0600 caddy:caddy. The cert is
-    #    world-readable (Postfix/Dovecot read it as root before dropping
-    #    privileges); the private key is restricted to root (0640) so it is not
-    #    exposed to other local users.
+    # 1) Propagate Caddy's renewed certificate into the distribution dir stalwart
+    #    reads. Caddy stores certs 0600 caddy:caddy. The cert is world-readable;
+    #    the private key is restricted to root (0640) and stalwart-cert-perm
+    #    regrants it to the "stalwart" group so it is not exposed to other local
+    #    users.
     install -D -m 0644 "$SRC_CERT" "$DST_CERT"
     install -D -m 0640 "$SRC_KEY"  "$DST_KEY"
+    # Stalwart reads the key as user "stalwart" via %{file:...}%; regrant the
+    # group immediately so every sync leaves it readable (0640 root:stalwart)
+    # and webadmin config reload never fails with EACCES.
+    chgrp stalwart "$DST_KEY"
 
     # 2) TLSA 3 1 1 = SHA-256 of the certificate's SubjectPublicKeyInfo (SPKI),
     #    NOT the whole certificate. Matching type 1 = SHA-256 of the SPKI DER.
@@ -47,8 +55,24 @@
 
     # 4) Reload services so the changes take effect immediately.
     systemctl reload coredns.service || true
-    podman exec mailserver postfix reload || true
-    podman exec mailserver dovecot reload || true
+    # Stalwart reads its TLS certs ($certDir) via %{file:...}% placeholders at
+    # startup only; there is no signal-based reload (management API reload needs
+    # admin credentials we must not store). Restart it, but only when the cert
+    # actually changed (SPKI hash differs from the last applied run), so the
+    # daily timer and inotify events for unchanged certs do not drop connections.
+    # NOTE: stalwart.service `requires` this unit, so the restart must be issued
+    # with --no-block (async): a synchronous restart waits for stalwart to come
+    # back up, which in turn waits for this unit to finish — a deadlock. The state
+    # file is updated BEFORE the restart so the tlsa-update run that stalwart's
+    # required-activation re-triggers sees a matching hash and exits immediately.
+    if [ "$(cat "${stateFile}" 2>/dev/null || true)" != "$HEX" ]; then
+      printf '%s\n' "$HEX" > "${stateFile}"
+      # Restarting stalwart re-runs its Requires=tlsa-update dependency; guard
+      # with is-active so a boot-time run never races stalwart's initial start.
+      if systemctl is-active --quiet stalwart.service 2>/dev/null; then
+        systemctl --no-block restart stalwart.service || true
+      fi
+    fi
 
     echo "tlsa-update: TLSA set to $HEX"
   '';
@@ -57,6 +81,7 @@ in {
     # Ensure the distribution dir exists (Caddy does not write here).
     tmpfiles.rules = [
       "d ${distCertDir} 0755 root root - -"
+      "d /var/lib/tlsa-update 0755 root root - -"
     ];
 
     services.tlsa-update = {
@@ -64,7 +89,7 @@ in {
       after = ["caddy.service" "coredns.service"];
       partOf = ["coredns.service"];
       wantedBy = ["multi-user.target"];
-      path = with pkgs; [openssl coreutils gnused podman systemd];
+      path = with pkgs; [openssl coreutils gnused systemd];
       serviceConfig = {
         Type = "oneshot";
         ExecStart = "${syncScript}";

servers/hetzner/modules/services/virtualcam-checkout.patch

@@ -0,0 +1,32 @@
+--- a/src/app/api/checkout/route.ts
++++ b/src/app/api/checkout/route.ts
+@@ -7,6 +7,8 @@
+ 
+ export const dynamic = "force-dynamic";
+ 
++const BYPASS_PAYMENTS = process.env.BYPASS_PAYMENTS === "true";
++
+ export async function POST(req: Request) {
+   // Each checkout hits the payment provider, so cap order creation per user
+   // and per IP to prevent order spam and provider-API abuse.
+@@ -86,6 +88,20 @@
+     const paid = await prisma.order.findUnique({
+       where: { id: order.id },
+       include: { license: true },
++    });
++    return NextResponse.json({
++      ok: true,
++      orderId: order.id,
++      licenseKey: paid?.license?.key ?? null,
++      tier: tier.slug,
++    });
++  }
++
++  if (BYPASS_PAYMENTS) {
++    await markOrderPaid({ orderId: order.id, providerRef: "bypass-test" });
++    const paid = await prisma.order.findUnique({
++      where: { id: order.id },
++      include: { license: true },
+     });
+     return NextResponse.json({
+       ok: true,

servers/hetzner/modules/services/virtualcam.nix

@@ -0,0 +1,287 @@
+{
+  pkgs,
+  lib,
+  unstablePkgs,
+  ...
+}: let
+  rev = "6e351ddc732421eb775eb9843ccd2389d6525edd";
+  apiRev = "390878d126185b21a14479f85c70134a155c61c6";
+
+  # Private repositories are fetched over SSH (port 2222). nix-daemon runs as
+  # root and uses /root/.ssh (identity materialized by the git-ssh-key unit),
+  # so the source ends up in the store without any Nix-native credentials.
+  src = builtins.fetchGit {
+    url = "ssh://git@git.severijnse.eu:2222/jory/virtualcam-website.git";
+    rev = rev;
+  };
+
+  srcPatched = pkgs.applyPatches {
+    name = "virtualcam-website-patched";
+    src = src;
+    patches = [./virtualcam-checkout.patch];
+  };
+
+  apiSrc = builtins.fetchGit {
+    url = "ssh://git@git.severijnse.eu:2222/jory/virtualcam-api.git";
+    rev = apiRev;
+  };
+
+  # Build the Next.js app entirely in Nix (offline npm deps from the lockfile).
+  app = unstablePkgs.buildNpmPackage {
+    pname = "virtualcam-website";
+    version = "0.1.0";
+    src = srcPatched;
+    npmDepsHash = "sha256-GodWQKtOtsLOjjiwzxun+wTPhtvjR2uCV91n+wiZHw4=";
+    nodejs = unstablePkgs.nodejs;
+
+    buildPhase = ''
+      runHook preBuild
+      npx prisma generate
+      npm run build
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+      mkdir -p $out
+      cp -r .next node_modules public prisma src package.json package-lock.json \
+        prisma.config.ts next.config.ts tsconfig.json postcss.config.mjs $out/
+      runHook postInstall
+    '';
+
+    APP_URL = "https://virtualcam.severijnse.eu";
+    GOOGLE_CLIENT_ID = "754775011707-c699m092tv3icmovhk5qa106v3q6eh7c.apps.googleusercontent.com";
+    # Only used to satisfy prisma generate / next build metadata resolution.
+    DATABASE_URL = "postgresql://virtualcam@localhost/virtualcam?host=/run/postgresql&schema=public";
+    # Use the nixpkgs-bundled Prisma engine so the sandboxed offline build does
+    # not need to download it from binaries.prisma.sh. Version matches ^7.8.0.
+    PRISMA_SCHEMA_ENGINE_BINARY = "${unstablePkgs.prisma-engines}/bin/schema-engine";
+    NEXT_TELEMETRY_DISABLED = "1";
+    CI = "true";
+  };
+
+  # License validation / admin API (Go/Fiber), serving on host port 3004.
+  # go.mod demands go 1.26.5 but the pinned nixpkgs only has 1.26.4; the code
+  # uses nothing newer, so the directive is relaxed to match the toolchain.
+  api = unstablePkgs.buildGoModule {
+    pname = "virtualcam-api";
+    version = "0.1.0";
+    src = apiSrc;
+    vendorHash = "sha256-uvHClXHw9ycoIf6qBZmV2O3CSyIxCgnSPCSALM07qg8=";
+    go = unstablePkgs.go_1_26;
+    postPatch = ''
+      sed -i 's/^go 1\.26\.5$/go 1.26.4/' go.mod
+    '';
+  };
+
+  dbUrl = "postgresql://virtualcam@localhost/virtualcam?host=/run/postgresql&schema=public";
+  # lib/pq parses the DSN differently from node-postgres: a hostname in the URL
+  # authority wins over a `host=` query param (so it would go over TCP and fail
+  # password auth), and lib/pq rejects unknown URL params like `schema`. Use a
+  # keyword DSN: unix-socket + peer auth + no SSL, matching the OS user.
+  apiDbUrl = "host=/run/postgresql user=virtualcam dbname=virtualcam sslmode=disable";
+
+  # Root-only runtime environment file holding the secrets both services need.
+  secretsFile = ../../secrets/secrets.yaml;
+  envFile = "/var/lib/virtualcam/environment";
+
+  # Materialize the license signing key and admin token from sops into a
+  # root-only file (0600). systemd reads environmentFiles before dropping
+  # privileges, so the service users never need to read it themselves.
+  writeSecrets = pkgs.writeShellScript "virtualcam-write-secrets" ''
+    set -euo pipefail
+    install -d -o virtualcam -g virtualcam -m 0750 /var/lib/virtualcam
+    : > "${envFile}"
+    chmod 0600 "${envFile}"
+    ${pkgs.sops}/bin/sops --decrypt --input-type yaml --output-type yaml ${secretsFile} |
+      ${pkgs.gnused}/bin/sed -nE \
+         's/^virtualcam_license_signing_key: (.*)/LICENSE_SIGNING_KEY=\1/p; s/^virtualcam_admin_token: (.*)/ADMIN_TOKEN=\1/p; s/^virtualcam_smtp_pass: (.*)/SMTP_PASS=\1/p; s/^virtualcam_google_OAuth_secret: (.*)/GOOGLE_CLIENT_SECRET=\1/p; s/^shkeeper_api_key: (.*)/SHKEEPER_API_KEY=\1/p' \
+        >> "${envFile}"
+  '';
+
+  # Materialize the nix-daemon's SSH identity so private-repo fetches keep
+  # working after the one-off bootstrap copy in /root/.ssh.
+  writeGitSshKey = pkgs.writeShellScript "git-ssh-key" ''
+    set -euo pipefail
+    install -d -m 0700 /root/.ssh
+    ${pkgs.sops}/bin/sops --decrypt --extract '["git_ssh_key_b64"]' \
+      --input-type yaml --output-type yaml ${secretsFile} |
+      ${pkgs.coreutils}/bin/base64 -d > /root/.ssh/id_ed25519
+    chmod 0600 /root/.ssh/id_ed25519
+    cat > /root/.ssh/known_hosts <<'EOF'
+    git.severijnse.eu ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAzIVo9Jdp8kwnWmTn26Fj68baJjwDphYw/0HTH5BzYY
+    [git.severijnse.eu]:2222 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGcPKlvdxApYRCZuQxrljEdETrxAu0JsEJ+28zFaMfA3P69J9vEkqqAZZ0Uq3rtE8ZqPBxeDh7JYn2XChb+QVGjOHVWMCWqcdD43ws26eY91aHGYnO2kW+sKSxIh803tdifIp2q8VRnaHVXcLHLx2rmwmvqIfMLnEEShxAt4Q846DianBoA7S8heoBvMLi8AAO09sVEgMPWm8ULkoRq0poHslsKkdYKlWzecP85tPo/tlCEjQq5cC5+xidH9TS4ILxVIXGUfErZzbWZq5LtaRmqiU5U1+zzdb83oJvHFtTZ8wHwZX+VE4jZzKw7wdFPk+dJZp/3ZAVva6fLSjlpyyDFNWMcf2/18Gs6YioD1n2fr1dXruZiXNMnI19Dzyp+8yWIPbAeWyVx3BoCrcjIApEDMLMo7ut0l6RfNy80hYpY91VBrpsjSRLB4uCVDsqhTLPTrFUlu+9YY9lSHEUVLRylOLK54TbY089hbHOB/UdpyCiXpI0ds3ww1UmnDQSFjwNknc+wTXRvzEz+VsX2nAzfO+5NWq2jHNn4wH6GuAinAxSVg1A7Ub9VSmPelRiTzK6ZnplPm762MxcPFmo+y/5cszumUwypcyTS8MJIC57S/kabfsnjdt97K+9eT9X2LDZ2uIQgDS8qEgvbnvfXVmMoqakgBLYG7Embqc8BDzNsQ==
+    EOF
+    chmod 0600 /root/.ssh/known_hosts
+    test -f /root/.ssh/id_ed25519.pub || ${pkgs.openssh}/bin/ssh-keygen -y -f /root/.ssh/id_ed25519 > /root/.ssh/id_ed25519.pub
+  '';
+
+  # The repo's seed uses tsx (a devDependency buildNpmPackage drops) plus the
+  # "@/..." path alias. nixpkgs ships tsx, which honours tsconfig paths, so we
+  # add it to the service path rather than working around the missing dep.
+  seed = pkgs.writeShellScript "virtualcam-seed" ''
+    set -euo pipefail
+    export DATABASE_URL="${dbUrl}"
+    export PRISMA_SCHEMA_ENGINE_BINARY="${unstablePkgs.prisma-engines}/bin/schema-engine"
+    export HOME=/var/lib/virtualcam
+    cd ${app}
+    tsx prisma/seed.ts
+  '';
+
+  # One shared PostgreSQL server (existing system postgres). Each service gets
+  # its own database + role. virtualcam authenticates over the Unix socket via
+  # peer auth: the systemd services run as OS user `virtualcam`, which matches
+  # the database role `virtualcam`, so no password is stored anywhere.
+  migrate = pkgs.writeShellScript "virtualcam-migrate" ''
+    set -euo pipefail
+    export DATABASE_URL="${dbUrl}"
+    # Use the local Prisma engine; no network download needed at runtime.
+    export PRISMA_SCHEMA_ENGINE_BINARY="${unstablePkgs.prisma-engines}/bin/schema-engine"
+    cd ${app}
+    ./node_modules/.bin/prisma migrate deploy
+  '';
+in {
+  users = {
+    users.virtualcam = {
+      isSystemUser = true;
+      group = "virtualcam";
+      description = "virtualcamera services user";
+    };
+    groups.virtualcam = {};
+  };
+
+  services.postgresql = {
+    ensureDatabases = ["virtualcam"];
+    ensureUsers = [
+      {
+        name = "virtualcam";
+        ensureDBOwnership = true;
+      }
+    ];
+  };
+
+  systemd = {
+    services = {
+      git-ssh-key = {
+        description = "Materialize nix-daemon git SSH key from sops";
+        wantedBy = ["multi-user.target"];
+        environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
+        serviceConfig = {
+          Type = "oneshot";
+          Environment = ["HOME=/root"];
+          ExecStart = "${writeGitSshKey}";
+        };
+      };
+
+      virtualcam-secrets = {
+        description = "Materialize virtualcam secrets from sops";
+        wantedBy = ["multi-user.target"];
+        # The age key lives in /etc/age/keys.txt; the service must know where it
+        # is and needs a HOME for age to report its user config directory.
+        environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
+        serviceConfig = {
+          Type = "oneshot";
+          Environment = ["HOME=/root"];
+          ExecStart = "${writeSecrets}";
+        };
+      };
+
+      virtualcam-migrate = {
+        description = "Virtualcam Prisma migrations";
+        after = ["postgresql.service"];
+        requires = ["postgresql.service"];
+        wantedBy = ["multi-user.target"];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "virtualcam";
+          Group = "virtualcam";
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+          ExecStart = "${migrate}";
+        };
+      };
+
+      virtualcam-seed = {
+        description = "Virtualcam catalog seed";
+        after = ["virtualcam-migrate.service"];
+        requires = ["virtualcam-migrate.service"];
+        wantedBy = ["multi-user.target"];
+        path = [unstablePkgs.nodejs unstablePkgs.tsx];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "virtualcam";
+          Group = "virtualcam";
+          ExecCondition = "!/var/lib/virtualcam/.seeded";
+          ExecStart = "${seed}";
+          ExecStartPost = "${pkgs.coreutils}/bin/touch /var/lib/virtualcam/.seeded";
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+        };
+      };
+
+      virtualcam = {
+        description = "Virtualcamera website (Next.js)";
+        after = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
+        requires = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
+        wantedBy = ["multi-user.target"];
+        path = [unstablePkgs.nodejs];
+        serviceConfig = {
+          User = "virtualcam";
+          Group = "virtualcam";
+          WorkingDirectory = "${app}";
+          EnvironmentFile = [envFile];
+          ExecStart = "${app}/node_modules/.bin/next start -p 3001 -H 127.0.0.1";
+          Restart = "on-failure";
+          RestartSec = 5;
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+          Environment = [
+            "DATABASE_URL=${dbUrl}"
+            "APP_URL=https://virtualcam.severijnse.eu"
+            "GOOGLE_CLIENT_ID=754775011707-c699m092tv3icmovhk5qa106v3q6eh7c.apps.googleusercontent.com"
+            "PAYMENTS_MODE=shkeeper"
+            "BYPASS_PAYMENTS=false"
+            "SHKEEPER_URL=https://pay.severijnse.eu"
+            "SHKEEPER_CRYPTO=BTC"
+            "SHKEEPER_FIAT=USD"
+            "ADMIN_EMAILS=jory@severijnse.eu"
+            "SMTP_HOST=mail.severijnse.eu"
+            "SMTP_PORT=587"
+            "SMTP_USER=no-reply@severijnse.eu"
+            "SMTP_FROM=no-reply@severijnse.eu"
+            "NODE_ENV=production"
+            "NEXT_TELEMETRY_DISABLED=1"
+            "HOME=/var/lib/virtualcam"
+          ];
+        };
+      };
+
+      virtualcam-api = {
+        description = "Virtualcamera license API (Go/Fiber)";
+        after = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
+        requires = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
+        wantedBy = ["multi-user.target"];
+        serviceConfig = {
+          User = "virtualcam";
+          Group = "virtualcam";
+          WorkingDirectory = "${api}";
+          EnvironmentFile = [envFile];
+          ExecStart = "${api}/bin/virtualcam-api";
+          Restart = "on-failure";
+          RestartSec = 5;
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+          Environment = [
+            "PORT=3004"
+            "DATABASE_URL=${apiDbUrl}"
+            "CORS_ORIGINS=https://virtualcam.severijnse.eu"
+            "HOME=/var/lib/virtualcam"
+          ];
+        };
+      };
+    };
+  };
+
+  # Served behind Caddy (virtualHosts wired in caddy.nix).
+  networking.firewall.allowedTCPPorts = [];
+}

servers/hetzner/modules/services/wireguard.nix

@@ -1,16 +1,40 @@
-{...}: {
+{pkgs, ...}: let
+  # sops-encrypted secrets (single file holds all service secrets), same as backup.nix.
+  secretsFile = ../../secrets/secrets.yaml;
+  # Root-only env file wg-easy reads the admin password from (0600 root).
+  wgEnvFile = "/var/lib/wg-easy/environment";
+
+  # Materialize the wg-easy admin password from sops into a root-only env file,
+  # so the secret never lands in the Nix store.
+  writeSecrets = pkgs.writeShellScript "wg-easy-write-secrets" ''
+    set -euo pipefail
+    mkdir -p "$(dirname ${wgEnvFile})"
+    PASSWORD="$(${pkgs.sops}/bin/sops \
+      --decrypt --extract '["wg_admin_password"]' \
+      --input-type yaml --output-type yaml ${secretsFile} | tr -d '\n')"
+    printf 'INIT_PASSWORD=%s\n' "$PASSWORD" > "${wgEnvFile}"
+    chmod 0600 "${wgEnvFile}"
+  '';
+in {
+  boot.kernelModules = ["ip6table_nat"];
+
   virtualisation.oci-containers.containers.wg-easy = {
-    image = "ghcr.io/wg-easy/wg-easy:latest";
+    image = "ghcr.io/wg-easy/wg-easy:15";
     autoStart = true;
     volumes = [
       "/home/admin/config:/etc/wireguard:Z"
     ];
+    environmentFiles = [wgEnvFile];
     environment = {
-      WG_HOST = "severijnse.eu";
-      PASSWORD_HASH = "$2a$12$b3n4drXgS3B6ubMZxxjPUOQ1XktZ1EuDwm4AIdVulhtoD7b1.WQGC";
-      WG_DEFAULT_ADDRESS = "10.8.0.x";
-      WG_DEFAULT_DNS = "1.1.1.1";
-      WG_PORT = "51820";
+      INSECURE = "true";
+      INIT_ENABLED = "true";
+      INIT_USERNAME = "admin";
+      INIT_HOST = "severijnse.eu";
+      INIT_PORT = "51820";
+      INIT_DNS = "1.1.1.1,2606:4700:4700::1111";
+      INIT_IPV4_CIDR = "10.8.0.0/24";
+      INIT_IPV6_CIDR = "fd10:8::/64";
+      INIT_ALLOWED_IPS = "0.0.0.0/0, ::/0";
     };
     extraOptions = [
       "--cap-add=NET_ADMIN"
@@ -19,4 +43,38 @@
       "--network=host"
     ];
   };
+
+  # Materialize the wg-easy admin password from sops before the container starts.
+  systemd.services.wg-easy-secrets = {
+    description = "Materialize wg-easy admin password from sops";
+    wantedBy = ["multi-user.target"];
+    # The age key lives in /etc/age/keys.txt; the service must know where it is
+    # and needs a HOME for age to report its user config directory.
+    environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
+    serviceConfig = {
+      Type = "oneshot";
+      Environment = ["HOME=/root"];
+      ExecStart = "${writeSecrets}";
+    };
+  };
+
+  systemd.services."podman-wg-easy" = {
+    requires = ["wg-easy-secrets.service"];
+    after = ["wg-easy-secrets.service"];
+  };
+
+  systemd.services.wg-nat66 = {
+    description = "NAT66 for WireGuard IPv6";
+    after = ["network.target" "podman-wg-easy.service"];
+    wants = ["podman-wg-easy.service"];
+    wantedBy = ["multi-user.target"];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    script = ''
+      ${pkgs.iptables}/bin/ip6tables -t nat -C POSTROUTING -s fd10:8::/64 -o enp1s0 -j MASQUERADE 2>/dev/null || \
+        ${pkgs.iptables}/bin/ip6tables -t nat -A POSTROUTING -s fd10:8::/64 -o enp1s0 -j MASQUERADE
+    '';
+  };
 }

servers/hetzner/secrets/secrets.yaml

@@ -0,0 +1,49 @@
+shkeeper_api_key: ENC[AES256_GCM,data:kUymtefYJOllK5cxKHhD3HnXO/uo1w==,iv:OH9ZaQJZsFfo00BIl5MlRw1XBzHUmFi1FW54/J6O/8Q=,tag:HQJyuEoz5fsDi7iMq3PEzw==,type:str]
+restic_password: ENC[AES256_GCM,data:OHQlxUpNnTqMQm6A/o3ID/3F91NpVKOrsFYdLqrtI4vp+6TWHR8=,iv:bBd+gfi78lgTaTv0QUWYGQWPsurrzx90VvOzA2d2euA=,tag:qsHSdAHx1auwdZCgn2O5Qw==,type:str]
+b2_key_id: ENC[AES256_GCM,data:4B9rvg06baH6aNiT,iv:Qk1ToF3lMYLTrZdzpfaoGVzdiKYs492w9fzn4/TbNfQ=,tag:djxfqwgNZmRGqvteTX9G3w==,type:str]
+b2_application_key: ENC[AES256_GCM,data:xGAbBqx+6ErK7sy3FR0yza6mJU5oc5uQQGcwKtcPp1Ef4XEVd8do/wN7,iv:CojcoykDRBfvN8yqMMrPJq9mtAwxyswqXXVBKPupxDs=,tag:BkfFHAGJvtCDUpFoxXVP3Q==,type:str]
+stalwart_admin_hash: ENC[AES256_GCM,data:WqRfogOC+LrlSmg8/A4kLaUVJRdGgfrCnbuAzwyf4SMed37vsialN5QPR1IxiY02ueqz12dZ1VObittLMGxK+tUIIAwv3+F3uyKboDSyP9+wGAhBkcwTFTMjc6RMzaJ5VHW4OGaSvijiMQ==,iv:jvzoEI8evtFxuZ7W4P3nkQ0KwccVLj0j/s0s/MMXmIo=,tag:KopbV++Y/aABh1c8zfo1dw==,type:str]
+wg_admin_password: ENC[AES256_GCM,data:X/+YJoNoFFU7P/2HCpTI,iv:FXDS3xuFB9jxjpQhsX2EKn0YFVzL23c2sZO6RqpKTiw=,tag:ccyQVzqNc1HEY0rxJSLlKw==,type:str]
+virtualcam_license_signing_key: ENC[AES256_GCM,data:zk8YDER9d9mBOdXgEKKC6dubSMfz6UMMb758ZC9SvjkYIv+R2EVd+CJwL93MCs0l62lUDxoBDCgTLbKM55kcaThkkqfgZCYPrndXevYZ7kquNQEMkoYg/pyGAWhAQ9trvT5QXfBES9UDdYi6J9taCVKCofutiMEdoJM8olgUwKsP4i+fbt3idYkdWNGelastPnvTuUK/M3/dOKR5eheiy5oIntG+sECOz/2eKHJ8cU3ja18uAEwcpd9VojuOa0fvfui8AujQcMZNSMv8VsLGnoR854iX3g1MvHkTBSzv3qaDzRC54MueXYR4WOzm0o+yv6Otmw1n9trWtk+WLUimmiKxDdW6nSFaddeisWzy47+QYzrSsd7iovNYEIHnW4kpnT5W2g5lwaJwA0J0pBOqvV9gziv8oBNeEbLurBxi78KwHBqG,iv:svx+hC4tS26xqI05+yo6N2d3uHzO7ULM0e76dMbShjs=,tag:EJ9fjHEi1wUhLkxy8sexww==,type:str]
+virtualcam_admin_token: ENC[AES256_GCM,data:JC5zSFBCoEapxv/Kcs4VXk3HAkiqwXVrCv1rlqFsXVx3LGspnQdzjOIahlM=,iv:GRVfMNGkwC3xXR00ww29rFbhTrpuViad+H05yDhR9BA=,tag:OBnxBD/8KU4mn17FFTcdng==,type:str]
+git_ssh_key_b64: ENC[AES256_GCM,data:LaUcR005w9iCd/DifDhzLM7tU4AuTiFM802gMU1HMtvFN2iSffucaFJWxpKv2zFxDxz0QpA3XwM1M5yJC4K3YFMxPkPXcWLYN4yHcc1yXuoB3SseQXjcJnPMqZvJfVYhHep+6CVVlVDQs1Y+miBvbAhrvCEblwUu5LhD7yvoJLwTnEpT15WGDvqezzYKzNSyQGNRa8KPjsD/7gJN3RyUPzKo6GsrIp8d8eOTuCZwEcJVDKE0ACXs4V6cd+akqIWZkNPyilnMTuK5cqg2YuqSpwu9NncKED6SjmPiozPKKgTVaWhNCRJUTBfIw85fdXizvq7PNbNRxxBsscAWd0ixXiLqg1RTipfv85wL4Nru3r3qcQGAWmOpTHKW5fA9kR4ygdNxEVkOgQ70m9U0wev+vdIJUoRYj09Ikobej370NKppOP6+oeEDE7NpkFH7tzRk1oHtwoc1dwP9Kq1T8XdxttCKFyb1pAJ6gMdMThCfJlC9/Uu/WYbYVu5bCj6BAmrdrmhbPrsEOylBeC9j9RhyRrFEqLd8AZ01Zg8oKcj9WtCWEMf9Y4biv2xE43JbBGRvLAx3sH9MRplu2hN2W/bRxa/EPj9GHgbUjScDBkm/QVbMeV+QOgi09sCHDTeSX3BHdZ8ZLNjuNUwYnWZq4lPix0iYS5MufiixNdQmncPeHq0UuBtVifEl2Jmjq1COLyeAQe47AKQ5zDjT91xtBRZVZEVk0kGBYHtBEJ9ezPwckSw=,iv:cbIpwwhmyJVvvTMB95YBzt+RQLOji6yJh+SpcJc3Q1Y=,tag:t8WxU1IPDfkB8muTdBLUhg==,type:str]
+virtualcam_smtp_pass: ENC[AES256_GCM,data:D5kDTQP9/YV5BjAWeTn0G2zKe+s=,iv:7dZJprrs0S2ECtbDQuvhUWsU0Xzpld5/XZqdbyefg0o=,tag:uWe811KAisBJ5M1EhJjrcQ==,type:str]
+virtualcam_google_OAuth_secret: ENC[AES256_GCM,data:kPJIP4iH2vFo+VRojvhjzrQhDWidFq7kjC7ZaFcwJ7dNHlo=,iv:qKWClwSIRPU2Xz50bGYqU+eMR/iFxqnaKkb18eL3aMU=,tag:78jGQgL1myDBMiZP8GZ80g==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1yd59qp5km4cxt99rlfjehnsucrjn9lmj0su4h3avhf6vrtjvnyjqstldl5
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycGNTU1cvbDFhMXRmb2xj
+            Sm83dEdRV2Nsc28vWGwvWDRoYUxnaC9BVWdBClh0a1lLMEhIa0NGVmNQL0NTT05y
+            Y1g1eU1jUEo3bFg5OWw2a1JNdkozanMKLS0tIFVXL1hueGcwQ2ZuMXg0alVxSEZM
+            eDhBZ1UweEJ1UlZJdzB0RDhiRG1MeDgKLm2QHJAi++C5C86DDrl7dM0MSpYu11mn
+            PhD9ElDJO9dFVCh+X+CSJwKEslR1aAraE2iQSHHgWxbMw7MB6AA9uQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age15rhqdpwejyf3r6ww70qgv6hqmkpsqraakn26kc49wlauhaceaeqsmuwrdd
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPdUQ3aVYxWElRc3ZBVENn
+            L0dud1ViVi9uZHJ6NXcxN0VVd2VlNGZhUXdnClZpRWhJcC9sYjBxVk1kdkdGZmVE
+            QS9LL2ZtNnJIdmhpa2NUTlJqQk5jSUkKLS0tIER2emlCbkJ1ZFdHdmdPNndZT09I
+            MFhYTUpJaUhDVmlrVzNocWhRZ0t1Q00KV1rgDAOoqlzEuO7xoo2ZYL20dF3f1pCj
+            bNllMkJ0u+hSZin6aUIRV31ExCYtgivDVD3Jx09PKwuYqkINU6ui8A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1xekdrkjfu82hkxltydm72tllzgayyvfavvudeks3xjuujm5wt5hq6g55v0
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuYWlFTTlHMWFaeGEvNGhz
+            YVhEaVdOWXAvNTZrMjhRUWo1WUtNRWlyTGd3CkkrYnFIYnJncDdQZzdPcWR2UDJB
+            TUw3ckF1eEdlTDA3SXVsdmVKMUpDamMKLS0tIGVQK2hsWUlwTUhzSG5keFROWEMz
+            MHJrVVpDYWdJNmxtUkozSzR4Nmt3R28KrhYi830HUFAPfg8WvPad7BAuNe1mYOWt
+            WEFIquuX/H/N+y/7uQcBDbvnBzyropE1hW8aNrxSKMeawvQZWNXkZA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-19T10:30:19Z"
+    mac: ENC[AES256_GCM,data:JOWFguh2iE4+OyU00l+2gn7GWv2yej37OLLhVVO6Yb7Kv4vgk4NhZKJLqopteQK1o6dPeWbeIydHG1Qmef0Sm0NmCyi7i6BCTplVizf/puaGjpkU5YAs90+HiEa9cKlNQ5brfNH3YecJeN9dK+d7o8bMO/xPv3raqFMBBoVtgu4=,iv:A7kCOzyWwiVaXKVMKXQ4j86zz2f65wAjOtaVEuLa81I=,tag:bcbPsmRRFRHX7qi4WdYb0g==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

system/nix/default.nix

@@ -19,6 +19,16 @@
   in {
     package = pkgs.lix;
 
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 14d";
+    };
+
+    optimise = {
+      automatic = true;
+    };
+
     # pin the registry to avoid downloading and evaling a new nixpkgs version every time
     registry = lib.mapAttrs (_: v: {flake = v;}) flakeInputs;