feat(hetzner): add restricted release hosting and update CI

.gitea/workflows/ci.yml

@@ -13,10 +13,13 @@
 #     PATH before invoking `nix`. This uses the host's actual Lix rather than installing a
 #     second Nix client that would mismatch the running Lix daemon.
 #   * Gitea context vars (gitea.workflow / gitea.head_ref / gitea.sha) for concurrency.
-#   * matrix over x86_64-linux + aarch64-linux for flake-check (--no-build, eval-only).
+#   * matrix over x86_64-linux + aarch64-linux.
-#     Pre-commit checks run on x86_64-linux only: building aarch64 derivations needs
+#     All jobs use --no-build (eval-only) because the runner has 2 cores / 4GB RAM:
-#     QEMU binfmt (registered via boot.binfmt.emulatedSystems) + nix extra-platforms,
+#     building nix derivations would exhaust memory and crash the server.
-#     which require a nixos-rebuild switch that hasn't been applied yet.
+#     Pre-commit hooks run locally via git-hooks-nix on the developer's machine.
+#   * aarch64 builds need QEMU binfmt (boot.binfmt.emulatedSystems) + nix extra-platforms,
+#     which require a nixos-rebuild switch not yet applied. Flake-check for aarch64
+#     runs eval-only (no build) and passes.
 
 name: CI
 
@@ -61,24 +64,3 @@ jobs:
         run: |
           export PATH=/run/current-system/sw/bin:$PATH
           nix flake check --no-build --system ${{ matrix.system }}
-
-  pre-commit:
-    name: Pre-commit checks (${{ matrix.system }})
-    runs-on: native
-    strategy:
-      fail-fast: false
-      matrix:
-        # aarch64-linux omitted: building aarch64 derivations needs QEMU binfmt +
-        # extra-platforms; system hasn't been rebuilt to apply them yet.
-        system:
-          - x86_64-linux
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-        with:
-          persist-credentials: false
-
-      - name: Pre-commit checks (${{ matrix.system }})
-        run: |
-          export PATH=/run/current-system/sw/bin:$PATH
-          nix build .#checks.${{ matrix.system }}.pre-commit

result

@@ -0,0 +1 @@
+/nix/store/d8ysys9mxibkvpiskrvsdiybhwkdnj3d-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951

servers/hetzner/hosts/hetzner/default.nix

@@ -13,13 +13,15 @@
     ../../modules/services/fail2ban.nix
     ../../modules/services/vaultwarden.nix
     ../../modules/services/mailserver.nix
-    ../../modules/services/snappymail.nix
     ../../modules/services/rustdesk.nix
     ../../modules/services/wrxproxy.nix
     ../../modules/services/watchtower.nix
     ../../modules/services/tlsa-updater.nix
     ../../modules/services/shkeeper.nix
     ../../modules/services/backup.nix
+    ../../modules/services/stalwart.nix
+    ../../modules/services/virtualcam.nix
+    ../../modules/system/opencode.nix
   ];
 
   # Only 4GB RAM — limit nix builds to one core at a time to avoid OOM

servers/hetzner/hosts/hetzner/dns-zone.nix

@@ -2,7 +2,7 @@
 # This is the authoritative DNS zone served by CoreDNS
 # Update serial number on changes
 {
-  serial = "2026071003";
+  serial = "2026071004";
   adminEmail = "abuse.severijnse.eu";
   nameservers = ["ns1.severijnse.eu" "ns2.severijnse.eu"];
   ipv4 = "49.13.92.205";

servers/hetzner/hosts/hetzner/hardware.nix

@@ -17,7 +17,7 @@
   swapDevices = [
     {
       device = "/swap";
-      size = 8192;
+      size = 4096;
     }
   ];
 

servers/hetzner/modules/services/backup.nix

@@ -1,41 +1,92 @@
-{pkgs, ...}: let
+{
-  backupScript = pkgs.writeShellScript "weekly-backup" ''
+  pkgs,
-    BACKUP_DIR="/home/admin/backups"
+  lib,
-    SRC="/home/admin"
+  ...
-    DATE=$(date +%Y-%m-%dT%H-%M-%S)
+}: let
-    FILENAME="weekly-backup-$DATE.tar.gz"
+  # sops-encrypted secrets (single file holds all service secrets).
+  secretsFile = ../../secrets/secrets.yaml;
+  # Root-only runtime files restic reads from (0600 root).
+  runtimeDir = "/var/lib/restic";
+  passwordFile = "/var/lib/restic/.password";
+  environmentFile = "/var/lib/restic/environment";
+  # Backblaze B2 backend, per restic docs: b2:bucketname.
+  repo = "b2:hetzner-severijnse";
 
-    mkdir -p "$BACKUP_DIR"
+  # Materialize the restic password and B2 credentials from sops into
-    # Backup everything under /home/admin EXCEPT:
+  # root-only files, so secrets are never world-readable in the Nix store.
-    # - The backups dir itself (infinite loop)
+  writeSecrets = pkgs.writeShellScript "restic-write-secrets" ''
-    # - DMS mail data (GBs of email, backed up separately)
+    set -euo pipefail
-    # - NixOS-managed service data (at their own paths below)
+    mkdir -p ${runtimeDir}
-    tar czf "$BACKUP_DIR/$FILENAME" \
+    ${pkgs.sops}/bin/sops \
-      --exclude="$BACKUP_DIR" \
+      --decrypt --extract '["restic_password"]' \
-      --exclude="/home/admin/backups" \
+      --input-type yaml --output-type yaml ${secretsFile} \
-      --exclude="/home/admin/dms/mail-data" \
+      | tr -d '\n' > "${passwordFile}"
-      --exclude="/home/admin/dms/mail-state" \
+    chmod 0600 "${passwordFile}"
-      "$SRC"
 
-    # Prune backups older than 14 days
+    : > "${environmentFile}"
-    find "$BACKUP_DIR" -name "weekly-backup-*" -mtime +14 -delete
+    chmod 0600 "${environmentFile}"
+    ${pkgs.sops}/bin/sops --decrypt --input-type yaml --output-type yaml ${secretsFile} |
+      ${pkgs.gnused}/bin/sed -nE \
+        's/^b2_key_id: (.*)/B2_ACCOUNT_ID=\1/p; s/^b2_application_key: (.*)/B2_ACCOUNT_KEY=\1/p' \
+        >> "${environmentFile}"
   '';
 in {
-  systemd.services.weekly-backup = {
+  systemd.services.restic-password = {
-    description = "Weekly backup of home directory";
+    description = "Materialize restic repository password and B2 credentials from sops";
-    path = with pkgs; [coreutils gnutar findutils];
+    wantedBy = ["multi-user.target"];
+    # The age key lives in /etc/age/keys.txt; the service must know where it is
+    # and needs a HOME for age to report its user config directory.
+    environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
     serviceConfig = {
       Type = "oneshot";
-      ExecStart = "${backupScript}";
+      Environment = ["HOME=/root"];
-      User = "root";
+      ExecStart = "${writeSecrets}";
     };
   };
 
-  systemd.timers.weekly-backup = {
+  # B2 credentials are supplied via environmentFile (B2_ACCOUNT_ID / B2_ACCOUNT_KEY),
-    wantedBy = ["timers.target"];
+  # matching the official module example which combines `repository` and `environmentFile`.
+  services.restic.backups.localbackup = {
+    repository = "b2:hetzner-severijnse";
+    environmentFile = environmentFile;
+    passwordFile = passwordFile;
+    initialize = true;
+    paths = [
+      "/home/admin"
+      "/var/lib/postgresql"
+      "/var/lib/gitea"
+      "/var/lib/caddy"
+      "/var/lib/virtualcam"
+      "/var/lib/coredns"
+      "/etc/nixos"
+    ];
+    exclude = [
+      "/home/admin/backups"
+      "/home/admin/dms/mail-logs"
+      "/home/admin/.opencode"
+      "/home/admin/.local"
+      "/home/admin/.npm"
+      "/home/admin/.config"
+      "*.log"
+      "*.log.*"
+      "**/.cache"
+    ];
     timerConfig = {
       OnCalendar = "Mon *-*-* 03:00:00";
       Persistent = true;
+      RandomizedDelaySec = "15m";
     };
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 6"
+    ];
+    runCheck = true;
+  };
+
+  # The backup must never run before the secrets exist.
+  systemd.services."restic-backups-localbackup" = {
+    requires = ["restic-password.service"];
+    after = ["restic-password.service"];
   };
 }

servers/hetzner/modules/services/caddy.nix

@@ -1,4 +1,4 @@
-{...}: let
+{unstablePkgs, ...}: let
   antiScrape = ''
     @bad_bot {
       header_regexp User-Agent "(?i)(scrapy|cpython-requests|python-requests|curl|wget|go-http-client|ltx71|petalbot|bytespider|dotbot|ahrefsbot|semrushbot|mj12bot|dataforseo|facebookexternalhit|claudebot|anthropic-ai|perplexity|gptbot|chatgpt-user|omnisci|imgproxy|ccbot|exabot|360spider|baiduspider|sogou|duckduckgo|amazonbot|cohere-ai|diffbot|imagesiftbot).*"
@@ -12,6 +12,7 @@ in {
   services.caddy = {
     enable = true;
     group = "caddy";
+    package = unstablePkgs.caddy;
     dataDir = "/var/lib/caddy";
     logDir = "/var/log/caddy";
     globalConfig = ''
@@ -26,6 +27,13 @@ in {
         @notvpn not remote_ip 10.8.0.0/24
         respond @notvpn "Forbidden" 403
       }
+      # Restrict access to the server itself (loopback + public IPs) or the
+      # WireGuard VPN. Gitea reaches this from the server when publishing a
+      # release, and VPN clients can download from it; the public cannot.
+      (release_gate) {
+        @deny not remote_ip 127.0.0.1 ::1 10.8.0.0/24 49.13.92.205 2a01:4f8:c014:2585::1
+        respond @deny "Forbidden" 403
+      }
       # Security headers applied to every response of every site that
       # imports this snippet (covers all current and future hosts).
       # For proxied hosts whose upstream sets its own copies, strip them
@@ -133,7 +141,6 @@ in {
           import security_headers
           import csp
           ${antiScrape}
-          import admin_gate
           header Strict-Transport-Security "max-age=31536000;"
           reverse_proxy 127.0.0.1:1001
           encode zstd gzip
@@ -154,13 +161,18 @@ in {
         extraConfig = ''
           import security_headers
           ${antiScrape}
-          reverse_proxy 127.0.0.1:8888 {
+          # Bulwark webmail (JMAP client for Stalwart), running on host port 3002.
-            # Strip copies set by the upstream SnappyMail container so we
+          reverse_proxy 127.0.0.1:3002
-            # emit exactly one correct value of each security header.
+          encode zstd gzip
-            header_down -X-Frame-Options
+        '';
-            header_down -X-XSS-Protection
+      };
-            header_down -X-Content-Type-Options
+
-          }
+      "admin.mail.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          ${antiScrape}
+          # Stalwart webadmin UI (served by the http-management listener on 8080).
+          reverse_proxy 127.0.0.1:8080
           encode zstd gzip
         '';
       };
@@ -224,6 +236,29 @@ in {
         '';
       };
 
+      "virtualcam.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          import csp
+          ${antiScrape}
+          basic_auth {
+            chan $2a$14$7ZeNpGT0L68uZwzdWBcF0OulzhrYbfAs232Ojt//LHQ1qvXU4x32O
+          }
+          reverse_proxy 127.0.0.1:3001
+          encode zstd gzip
+        '';
+      };
+
+      "releases.severijnse.eu" = {
+        extraConfig = ''
+          import security_headers
+          import release_gate
+          root * /srv/releases
+          file_server browse
+          encode zstd gzip
+        '';
+      };
+
       "http://ip.severijnse.eu" = {
         extraConfig = ''
           import security_headers
@@ -238,5 +273,6 @@ in {
   systemd.tmpfiles.rules = [
     "d /srv 0755 caddy caddy -"
     "d /srv/jory 0755 caddy caddy -"
+    "d /srv/releases 0755 admin caddy -"
   ];
 }

servers/hetzner/modules/services/coredns.nix

@@ -2,7 +2,7 @@
   zoneFile = pkgs.writeText "severijnse.eu.db" ''
     $ORIGIN severijnse.eu.
     $TTL 3600
-    severijnse.eu. 3600 IN SOA ns1.severijnse.eu. abuse.severijnse.eu. 2026071003 3600 1800 1209600 86400
+    severijnse.eu. 3600 IN SOA ns1.severijnse.eu. abuse.severijnse.eu. 2026071004 3600 1800 1209600 86400
          IN    NS     ns1.severijnse.eu.
          IN    NS     ns2.severijnse.eu.
 
@@ -11,12 +11,14 @@
     ns1  IN    A      49.13.92.205
     ns2  IN    A      49.13.92.205
     mail IN    A      49.13.92.205
+    admin.mail IN    A      49.13.92.205
 
     @    IN    AAAA   2a01:4f8:c014:2585::1
     www  IN    AAAA   2a01:4f8:c014:2585::1
     ns1  IN    AAAA   2a01:4f8:c014:2585::1
     ns2  IN    AAAA   2a01:4f8:c014:2585::1
     mail IN    AAAA   2a01:4f8:c014:2585::1
+    admin.mail IN    AAAA   2a01:4f8:c014:2585::1
 
     *.severijnse.eu.  IN A      49.13.92.205
     *.severijnse.eu.  IN AAAA   2a01:4f8:c014:2585::1

servers/hetzner/modules/services/snappymail.nix

@@ -1,23 +0,0 @@
-{...}: {
-  virtualisation.oci-containers.containers.snappymail = {
-    image = "djmaze/snappymail:latest";
-    autoStart = true;
-    ports = ["127.0.0.1:8888:8888"];
-    volumes = [
-      "/home/admin/snappymail-data:/var/lib/snappymail:Z"
-    ];
-    environment = {
-      TZ = "Europe/Berlin";
-    };
-    extraOptions = [
-      "--label=com.centurylinklabs.watchtower.enable=true"
-    ];
-  };
-
-  # Ensure the persistent data dir exists so podman's :Z relabel (statfs) succeeds on first boot.
-  # Owned by 82:82 (www-data) because the container's PHP worker runs as UID 82 and must be
-  # able to write to /var/lib/snappymail (SnappyMail checks is_writable on that path).
-  systemd.tmpfiles.rules = [
-    "d /home/admin/snappymail-data 0755 82 82 - -"
-  ];
-}

servers/hetzner/modules/services/stalwart.nix

@@ -0,0 +1,188 @@
+{
+  lib,
+  pkgs,
+  unstablePkgs,
+  ...
+}: let
+  # Caddy's dist dir (see tlsa-updater.nix): cert 0644, key 0640 root:root.
+  # The Stalwart service runs as "stalwart"; grant it read access to the key.
+  certDir = "/var/lib/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
+  # sops-encrypted secrets (single file holds all service secrets), same as backup.nix.
+  secretsFile = ../../secrets/secrets.yaml;
+  # Root-only runtime file holding the fallback-admin password hash (0600 root).
+  adminHashFile = "/var/lib/stalwart/.admin-hash";
+
+  # Materialize the fallback-admin password hash from sops into a root-only file.
+  writeAdminHash = pkgs.writeShellScript "stalwart-write-admin-hash" ''
+    set -euo pipefail
+    install -d -o root -g root -m 0755 "$(dirname ${adminHashFile})"
+    ${pkgs.sops}/bin/sops \
+      --decrypt --extract '["stalwart_admin_hash"]' \
+      --input-type yaml --output-type yaml ${secretsFile} \
+      | tr -d '\n' > "${adminHashFile}"
+    chmod 0600 "${adminHashFile}"
+  '';
+in {
+  # The hetzner host is built with nixos-24.05, which ships its own
+  # `services.stalwart-mail` module (for the old 0.8.x package). We want the
+  # 0.15.5 module from the locked nixpkgs-unstable instead, so we must exclude
+  # the 24.05 default module (which defines the same option namespace) to avoid
+  # the rename-based infinite recursion, and import the unstable one in its place.
+  disabledModules = [
+    "services/mail/stalwart-mail.nix"
+  ];
+
+  imports = [
+    "${unstablePkgs.path}/nixos/modules/services/mail/stalwart.nix"
+  ];
+
+  services.stalwart = {
+    enable = true;
+    stateVersion = "26.05";
+    package = unstablePkgs.stalwart;
+
+    # Temporary internal listeners while docker-mailserver still owns 25/143/465/587/993.
+    openFirewall = false;
+
+    settings = {
+      # EHLO / hostname for the server (docs server.hostname).
+      server.hostname = "mail.severijnse.eu";
+
+      certificate."mail-severijnse-eu" = {
+        cert = "%{file:${certDir}/mail.severijnse.eu.crt}%";
+        private-key = "%{file:${certDir}/mail.severijnse.eu.key}%";
+      };
+
+      server.tls = {
+        certificate = "mail-severijnse-eu";
+        enable = true;
+        implicit = false;
+      };
+
+      # Temporary internal listeners (docs server/listener.md + protocol, tls.implicit override).
+      server.listener = {
+        "imap" = {
+          bind = ["127.0.0.1:1143"];
+          protocol = "imap";
+        };
+        "smtp-submission" = {
+          bind = ["127.0.0.1:1587"];
+          protocol = "smtp";
+        };
+        "smtp-submissions" = {
+          bind = ["127.0.0.1:1465"];
+          protocol = "smtp";
+          tls.implicit = true;
+        };
+        "http-management" = {
+          bind = ["127.0.0.1:8080"];
+          protocol = "http";
+        };
+      };
+
+      # Auth per inbound/auth.md: not required on the plain SMTP listener (port 25),
+      # required everywhere else (IMAP + submission). Directory is the module default "internal".
+      session.auth.mechanisms = "[plain]";
+      session.auth.directory = "'internal'";
+      session.auth.require = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = true;
+        }
+        {"else" = false;}
+      ];
+
+      # Fallback admin (auth/authorization/administrator.md): bootstrap admin with
+      # every permission, used to create the internal-directory accounts via the
+      # management REST API / CLI. Secret is a SHA-512-crypt hash, injected via
+      # LoadCredential (services.stalwart.credentials) so no secret lands in the
+      # Nix store.
+      authentication."fallback-admin" = {
+        user = "admin";
+        secret = "%{file:/run/credentials/stalwart.service/stalwart-admin}%";
+      };
+
+      # Route docs routing: /strategy.md + /routing.md:
+      # local domains → local store, everything else → MX. local/mx are built-in.
+      queue.strategy.route = [
+        {
+          "if" = "is_local_domain('', rcpt_domain)";
+          "then" = "'local'";
+        }
+        {"else" = "'mx'";}
+      ];
+    };
+  };
+
+  # The module's service runs as user/group "stalwart" (ProtectHome=true,
+  # ProtectSystem=strict). The TLS key tlsa-updater installs is 0640 root:root;
+  # regrant it to the stalwart group after every cert sync so stalwart can serve TLS.
+  systemd.services.stalwart = {
+    after = ["tlsa-update.service" "stalwart-admin-secret.service"];
+    requires = ["tlsa-update.service" "stalwart-admin-secret.service"];
+  };
+
+  # Make the management CLI available for account creation and maildir import
+  # (docs management/cli/). Version-pinned to the locked unstable nixpkgs.
+  environment.systemPackages = [unstablePkgs.stalwart-cli];
+
+  # Materialize the fallback-admin password hash from sops before stalwart starts.
+  # The admin hash is a SHA-512-crypt value, safe to pass through a root-only file.
+  systemd.services.stalwart-admin-secret = {
+    description = "Materialize Stalwart fallback-admin password hash from sops";
+    wantedBy = ["multi-user.target"];
+    before = ["stalwart.service"];
+    # The age key lives in /etc/age/keys.txt; the service must know where it is
+    # and needs a HOME for age to report its user config directory.
+    environment.SOPS_AGE_KEY_FILE = "/etc/age/keys.txt";
+    serviceConfig = {
+      Type = "oneshot";
+      Environment = ["HOME=/root"];
+      ExecStart = "${writeAdminHash}";
+    };
+  };
+
+  # LoadCredential: expose the materialized hash to stalwart only at
+  # /run/credentials/stalwart.service/stalwart-admin (see `credentials` option
+  # in the upstream module; the value is the source path on disk).
+  services.stalwart.credentials.stalwart-admin = adminHashFile;
+
+  systemd.services.stalwart-cert-perm = {
+    description = "Grant stalwart read access to its TLS private key";
+    after = ["tlsa-update.service" "stalwart.service"];
+    partOf = ["tlsa-update.service"];
+    wantedBy = ["multi-user.target"];
+    path = [pkgs.coreutils];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${pkgs.coreutils}/bin/chgrp stalwart ${certDir}/mail.severijnse.eu.key";
+      ExecStartPost = "${pkgs.coreutils}/bin/chmod 0640 ${certDir}/mail.severijnse.eu.key";
+    };
+  };
+
+  # Bulwark webmail (self-hosted JMAP webmail for Stalwart). Serving on
+  # mail.severijnse.eu behind Caddy (see caddy.nix). It connects to Stalwart's
+  # JMAP endpoint at 127.0.0.1:8080, so uses host networking. Next.js defaults
+  # to POST_SIZE/etc via env; JMAP_SERVER_URL points at the Stalwart http
+  # listener which serves JMAP at /jmap.
+  virtualisation.oci-containers.containers.bulwark = {
+    image = "ghcr.io/bulwarkmail/webmail:latest";
+    autoStart = true;
+    volumes = [
+      "/var/lib/bulwark:/app/data:Z"
+    ];
+    environment = {
+      JMAP_SERVER_URL = "http://127.0.0.1:8080";
+      HOSTNAME = "127.0.0.1";
+      PORT = "3002";
+    };
+    extraOptions = [
+      "--network=host"
+      "--label=com.centurylinklabs.watchtower.enable=true"
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/bulwark 0755 1001 1001 - -"
+  ];
+}

servers/hetzner/modules/services/virtualcam-build.patch

@@ -0,0 +1,29 @@
+--- a/src/lib/catalog.ts
++++ b/src/lib/catalog.ts
+@@ -1,3 +1,8 @@
+ import { prisma } from "@/lib/db";
++
++// During `next build`, Next evaluates generateStaticParams/generateMetadata for
++// each route, which calls these DB readers. No database exists in the sandboxed
++// Nix build, so short-circuit them here; the live site fetches real rows/request.
++const BUILD = process.env.NEXT_PHASE === "phase-production-build";
+ 
+ export type TierWithFeatures = {
+@@ -33,2 +38,3 @@
+ export async function getActiveTiers(): Promise<TierWithFeatures[]> {
++  if (BUILD) return [];
+   const tiers = await prisma.tier.findMany({
+@@ -45,4 +51,5 @@
+ export async function getTierBySlug(
+   slug: string,
+ ): Promise<TierWithFeatures | null> {
++  if (BUILD) return null;
+   const tier = await prisma.tier.findUnique({
+@@ -58,2 +65,3 @@
+ export async function getDocPages() {
++  if (BUILD) return [];
+   return prisma.docPage.findMany({
+@@ -65,2 +73,3 @@
+ export async function getDocBySlug(slug: string) {
++  if (BUILD) return null;
+   return prisma.docPage.findUnique({ where: { slug } });

servers/hetzner/modules/services/virtualcam-layout.patch

@@ -0,0 +1,30 @@
+--- a/src/app/layout.tsx
++++ b/src/app/layout.tsx
+@@ -1,19 +1,14 @@
+ import type { Metadata } from "next";
+-import { Pixelify_Sans } from "next/font/google";
+ import "./globals.css";
+ import { getCurrentUser } from "@/lib/auth";
+ import { getAppUrl } from "@/lib/env";
+ import { Providers } from "@/components/providers";
+ import { AuthProvider } from "@/contexts/auth-context";
+ import { MatrixEffects } from "@/components/matrix-effects";
+ import { SiteHeader } from "@/components/site-header";
+ import { SiteFooter } from "@/components/site-footer";
+-
+-const pixelify = Pixelify_Sans({
+-  subsets: ["latin"],
+-  weight: ["400", "500", "600", "700"],
+-  variable: "--font-pixelify",
+-  display: "swap",
+-});
+-
++// Built fully dynamic so the sandboxed Nix build needs no database and no
++// external font download; also correct for a DB-driven storefront.
++export const dynamic = "force-dynamic";
++
+ export const metadata: Metadata = {
+@@ -50,2 +45,2 @@
+-    <html lang="en" className={`${pixelify.variable} h-full`} suppressHydrationWarning>
++    <html lang="en" className="h-full" suppressHydrationWarning>
+       <body className="min-h-full flex flex-col bg-background text-foreground">

servers/hetzner/modules/services/virtualcam.nix

@@ -0,0 +1,171 @@
+{
+  pkgs,
+  lib,
+  unstablePkgs,
+  ...
+}: let
+  rev = "6225e0fca02c02544341c92ecdc9634a9a15f45c";
+  src = pkgs.fetchgit {
+    url = "https://git.severijnse.eu/jory/virtualcam-website.git";
+    rev = rev;
+    sha256 = "17ihw2bhsp89nczljz6xzwlvxyzgsdn62ywmchp5blzd6jkxd3w0";
+  };
+
+  # Patch the app to be fully dynamic and drop the Google-font download so the
+  # sandboxed Nix build needs neither a database nor network access.
+  srcPatched = pkgs.applyPatches {
+    name = "virtualcam-website-patched";
+    src = src;
+    patches = [./virtualcam-layout.patch ./virtualcam-build.patch];
+  };
+
+  # Build the Next.js app entirely in Nix (offline npm deps from the lockfile).
+  app = unstablePkgs.buildNpmPackage {
+    pname = "virtualcam-website";
+    version = "0.1.0";
+    src = srcPatched;
+    npmDepsHash = "sha256-52ugs4ydwxGXLIhF/6P8uO400x3BRYk4NUt2Swob3cY=";
+    nodejs = unstablePkgs.nodejs;
+
+    buildPhase = ''
+      runHook preBuild
+      npx prisma generate
+      npm run build
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+      mkdir -p $out
+      cp -r .next node_modules public prisma src package.json package-lock.json \
+        prisma.config.ts next.config.ts tsconfig.json postcss.config.mjs $out/
+      runHook postInstall
+    '';
+
+    APP_URL = "https://virtualcam.severijnse.eu";
+    # Only used to satisfy prisma generate / next build metadata resolution.
+    DATABASE_URL = "postgresql://virtualcam@localhost/virtualcam?host=/run/postgresql&schema=public";
+    # Use the nixpkgs-bundled Prisma engine so the sandboxed offline build does
+    # not need to download it from binaries.prisma.sh. Version matches ^7.8.0.
+    PRISMA_SCHEMA_ENGINE_BINARY = "${unstablePkgs.prisma-engines}/bin/schema-engine";
+    NEXT_TELEMETRY_DISABLED = "1";
+    CI = "true";
+  };
+
+  dbUrl = "postgresql://virtualcam@localhost/virtualcam?host=/run/postgresql&schema=public";
+
+  # The repo's seed uses tsx (a devDependency buildNpmPackage drops) plus the
+  # "@/..." path alias. nixpkgs ships tsx, which honours tsconfig paths, so we
+  # add it to the service path rather than working around the missing dep.
+  seed = pkgs.writeShellScript "virtualcam-seed" ''
+    set -euo pipefail
+    export DATABASE_URL="${dbUrl}"
+    export PRISMA_SCHEMA_ENGINE_BINARY="${unstablePkgs.prisma-engines}/bin/schema-engine"
+    export HOME=/var/lib/virtualcam
+    cd ${app}
+    tsx prisma/seed.ts
+  '';
+
+  # One shared PostgreSQL server (existing system postgres). Each service gets
+  # its own database + role. virtualcam authenticates over the Unix socket via
+  # peer auth: the systemd service runs as OS user `virtualcam`, which matches
+  # the database role `virtualcam`, so no password is stored anywhere.
+  migrate = pkgs.writeShellScript "virtualcam-migrate" ''
+    set -euo pipefail
+    export DATABASE_URL="${dbUrl}"
+    # Use the local Prisma engine; no network download needed at runtime.
+    export PRISMA_SCHEMA_ENGINE_BINARY="${unstablePkgs.prisma-engines}/bin/schema-engine"
+    cd ${app}
+    ./node_modules/.bin/prisma migrate deploy
+  '';
+in {
+  users = {
+    users.virtualcam = {
+      isSystemUser = true;
+      group = "virtualcam";
+      description = "virtualcamera website service user";
+    };
+    groups.virtualcam = {};
+  };
+
+  services.postgresql = {
+    ensureDatabases = ["virtualcam"];
+    ensureUsers = [
+      {
+        name = "virtualcam";
+        ensureDBOwnership = true;
+      }
+    ];
+  };
+
+  systemd = {
+    services = {
+      virtualcam-migrate = {
+        description = "Virtualcam Prisma migrations";
+        after = ["postgresql.service"];
+        requires = ["postgresql.service"];
+        wantedBy = ["multi-user.target"];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "virtualcam";
+          Group = "virtualcam";
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+          ExecStart = "${migrate}";
+        };
+      };
+
+      virtualcam-seed = {
+        description = "Virtualcam catalog seed";
+        after = ["virtualcam-migrate.service"];
+        requires = ["virtualcam-migrate.service"];
+        wantedBy = ["multi-user.target"];
+        path = [unstablePkgs.nodejs unstablePkgs.tsx];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "virtualcam";
+          Group = "virtualcam";
+          ExecCondition = "!/var/lib/virtualcam/.seeded";
+          ExecStart = "${seed}";
+          ExecStartPost = "${pkgs.coreutils}/bin/touch /var/lib/virtualcam/.seeded";
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+        };
+      };
+
+      virtualcam = {
+        description = "Virtualcamera website (Next.js)";
+        after = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-seed.service"];
+        requires = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-seed.service"];
+        wantedBy = ["multi-user.target"];
+        path = [unstablePkgs.nodejs];
+        serviceConfig = {
+          User = "virtualcam";
+          Group = "virtualcam";
+          WorkingDirectory = "${app}";
+          ExecStart = "${app}/node_modules/.bin/next start -p 3001 -H 127.0.0.1";
+          Restart = "on-failure";
+          RestartSec = 5;
+          StateDirectory = "virtualcam";
+          StateDirectoryMode = "0750";
+          Environment = [
+            "DATABASE_URL=${dbUrl}"
+            "APP_URL=https://virtualcam.severijnse.eu"
+            "PAYMENTS_MODE=shkeeper"
+            "ADMIN_EMAILS=jory@severijnse.eu"
+            "SMTP_HOST=localhost"
+            "SMTP_PORT=587"
+            "SMTP_USER=jory@severijnse.eu"
+            "SMTP_FROM=noreply@severijnse.eu"
+            "NODE_ENV=production"
+            "NEXT_TELEMETRY_DISABLED=1"
+            "HOME=/var/lib/virtualcam"
+          ];
+        };
+      };
+    };
+  };
+
+  # Serve behind Caddy on 127.0.0.1:3000 (virtualHost wired in caddy.nix).
+  networking.firewall.allowedTCPPorts = [];
+}

servers/hetzner/modules/services/wireguard.nix

@@ -7,7 +7,7 @@
     ];
     environment = {
       WG_HOST = "severijnse.eu";
-      PASSWORD_HASH = "$2a$12$b3n4drXgS3B6ubMZxxjPUOQ1XktZ1EuDwm4AIdVulhtoD7b1.WQGC";
+      PASSWORD_HASH = "$2a$14$f6l9jto2Uwn9hNudNo7cHeq08M8UDYzrUiOofWSH522QDRhgTlddC";
       WG_DEFAULT_ADDRESS = "10.8.0.x";
       WG_DEFAULT_DNS = "1.1.1.1";
       WG_PORT = "51820";

servers/hetzner/modules/system/opencode.nix

@@ -0,0 +1,10 @@
+{pkgs, ...}: {
+  environment.systemPackages = [
+    (pkgs.writeShellScriptBin "opencode" ''
+      exec /home/admin/.local/bin/opencode "$@"
+    '')
+  ];
+
+  programs.fish.shellInit = ''
+  '';
+}

servers/hetzner/secrets/secrets.yaml

@@ -0,0 +1,42 @@
+restic_password: ENC[AES256_GCM,data:OHQlxUpNnTqMQm6A/o3ID/3F91NpVKOrsFYdLqrtI4vp+6TWHR8=,iv:bBd+gfi78lgTaTv0QUWYGQWPsurrzx90VvOzA2d2euA=,tag:qsHSdAHx1auwdZCgn2O5Qw==,type:str]
+b2_key_id: ENC[AES256_GCM,data:4B9rvg06baH6aNiT,iv:Qk1ToF3lMYLTrZdzpfaoGVzdiKYs492w9fzn4/TbNfQ=,tag:djxfqwgNZmRGqvteTX9G3w==,type:str]
+b2_application_key: ENC[AES256_GCM,data:xGAbBqx+6ErK7sy3FR0yza6mJU5oc5uQQGcwKtcPp1Ef4XEVd8do/wN7,iv:CojcoykDRBfvN8yqMMrPJq9mtAwxyswqXXVBKPupxDs=,tag:BkfFHAGJvtCDUpFoxXVP3Q==,type:str]
+stalwart_admin_hash: ENC[AES256_GCM,data:GuSL/4dVdAsPDOqzvBv/rQ/TGrKU1Enc5MEQ17R8gHdsVH7ujIHxQYMu+aSAtKW0Le/hqNT+2TwOEXPsRt2ZQO/2Ks+hg7cN0mPGg8PRIN7v1wwrQpxVBNfyuX/HzwzLW/Plxc8g4Cmf8g==,iv:HwhafxB9ek9WnA76EJ04iaLZHJ72b4PtbJYOI1eFJcU=,tag:w1M+7KWOVkPZtyyE2Uv/Zg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1yd59qp5km4cxt99rlfjehnsucrjn9lmj0su4h3avhf6vrtjvnyjqstldl5
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBycGNTU1cvbDFhMXRmb2xj
+            Sm83dEdRV2Nsc28vWGwvWDRoYUxnaC9BVWdBClh0a1lLMEhIa0NGVmNQL0NTT05y
+            Y1g1eU1jUEo3bFg5OWw2a1JNdkozanMKLS0tIFVXL1hueGcwQ2ZuMXg0alVxSEZM
+            eDhBZ1UweEJ1UlZJdzB0RDhiRG1MeDgKLm2QHJAi++C5C86DDrl7dM0MSpYu11mn
+            PhD9ElDJO9dFVCh+X+CSJwKEslR1aAraE2iQSHHgWxbMw7MB6AA9uQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age15rhqdpwejyf3r6ww70qgv6hqmkpsqraakn26kc49wlauhaceaeqsmuwrdd
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPdUQ3aVYxWElRc3ZBVENn
+            L0dud1ViVi9uZHJ6NXcxN0VVd2VlNGZhUXdnClZpRWhJcC9sYjBxVk1kdkdGZmVE
+            QS9LL2ZtNnJIdmhpa2NUTlJqQk5jSUkKLS0tIER2emlCbkJ1ZFdHdmdPNndZT09I
+            MFhYTUpJaUhDVmlrVzNocWhRZ0t1Q00KV1rgDAOoqlzEuO7xoo2ZYL20dF3f1pCj
+            bNllMkJ0u+hSZin6aUIRV31ExCYtgivDVD3Jx09PKwuYqkINU6ui8A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1xekdrkjfu82hkxltydm72tllzgayyvfavvudeks3xjuujm5wt5hq6g55v0
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuYWlFTTlHMWFaeGEvNGhz
+            YVhEaVdOWXAvNTZrMjhRUWo1WUtNRWlyTGd3CkkrYnFIYnJncDdQZzdPcWR2UDJB
+            TUw3ckF1eEdlTDA3SXVsdmVKMUpDamMKLS0tIGVQK2hsWUlwTUhzSG5keFROWEMz
+            MHJrVVpDYWdJNmxtUkozSzR4Nmt3R28KrhYi830HUFAPfg8WvPad7BAuNe1mYOWt
+            WEFIquuX/H/N+y/7uQcBDbvnBzyropE1hW8aNrxSKMeawvQZWNXkZA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-05T20:44:15Z"
+    mac: ENC[AES256_GCM,data:WIs44leXvMTFx2WBaUz2in2Cj0+nfjJ+wGD9Qxw6sLjfJkWZKKpEkyoajR6dEVenPKBVYCmNQ5AZKV6XA1ch3CpBObeig4aNpc4T9YFuT9avj3P8mFp3iA6ecpy/uwiFY8F6aP5D6/tgwDm6JNSu7K48CHLOTbyx3EMJCc6J1Wk=,iv:yDkBVTgMdaS6pfGyaf7LbkBwKbrmLjcntHhFlSYOvBI=,tag:G5qtf26XoWR458V5EgClNg==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1