feat(hetzner): add netdata

- Update virtualcam website and API to latest repository revisions.
- Add Google OAuth client ID and secrets decryption for Google auth.
- Enable Shkeeper BTC/USD payment processing (`BYPASS_PAYMENTS=false`) and add service dependencies.
- Update Caddy `admin_gate` IP access rules and remove redundant `admin_gate` import from app proxy.
- Configure weekly automatic Nix garbage collection (`--delete-older-than 14d`) and nix store optimization.

result

@@ -1 +1 @@
-/nix/store/3rwh3r83dmni3kkfmfnrh36lv0pb2zxn-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951
+/nix/store/p2h0fr7k47yrx3x0qkr3rwsp5nf5bj30-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951

servers/hetzner/hosts/hetzner/default.nix

@@ -20,7 +20,7 @@
     ../../modules/services/backup.nix
     ../../modules/services/stalwart.nix
     ../../modules/services/virtualcam.nix
-    ../../modules/system/opencode.nix
+    ../../modules/services/netdata.nix
   ];
 
   # Only 4GB RAM — limit nix builds to one core at a time to avoid OOM

servers/hetzner/modules/services/caddy.nix

@@ -24,7 +24,7 @@ in {
     # Global Caddyfile snippets (shared across all virtual hosts).
     extraConfig = ''
       (admin_gate) {
-        @notvpn not remote_ip 10.8.0.0/24
+        @notvpn not remote_ip 127.0.0.1 ::1 10.8.0.0/24 fd10:8::/64 49.13.92.205 2a01:4f8:c014:2585::1
         respond @notvpn "Forbidden" 403
       }
       # Restrict access to the server itself (loopback + public IPs) or the
@@ -245,7 +245,6 @@ in {
           import security_headers
           import csp
           ${antiScrape}
-          import admin_gate
           reverse_proxy 127.0.0.1:5000
           encode zstd gzip
         '';

servers/hetzner/modules/services/netdata.nix

@@ -0,0 +1,32 @@
+{
+  config,
+  pkgs,
+  ...
+}: {
+  #
+  # Netdata
+  #
+  services.netdata = {
+    enable = true;
+    config = {
+      global = {
+        "memory mode" = "ram";
+        "debug log" = "none";
+        "access log" = "none";
+        "error log" = "syslog";
+      };
+      web = {
+        "bind to" = "127.0.0.1:19999";
+      };
+    };
+  };
+
+  services.caddy.virtualHosts."netdata.severijnse.eu" = {
+    extraConfig = ''
+      import security_headers
+      import csp
+      reverse_proxy 127.0.0.1:19999
+      encode zstd gzip
+    '';
+  };
+}

servers/hetzner/modules/services/stalwart.nix

@@ -262,7 +262,12 @@ in {
 
   systemd.services.stalwart-cert-perm = {
     description = "Grant stalwart read access to its TLS private key";
-    after = ["tlsa-update.service" "stalwart.service"];
+    # Belt-and-suspenders: tlsa-update already chgrps the key after every sync;
+    # this guarantees the group grant also exists at first boot, before stalwart
+    # starts (previously ordered after stalwart, so a fresh sync could leave a
+    # root:root key and webadmin reload would fail with EACCES).
+    after = ["tlsa-update.service"];
+    before = ["stalwart.service"];
     partOf = ["tlsa-update.service"];
     wantedBy = ["multi-user.target"];
     path = [pkgs.coreutils];

servers/hetzner/modules/services/tlsa-updater.nix

@@ -33,6 +33,10 @@
     #    users.
     install -D -m 0644 "$SRC_CERT" "$DST_CERT"
     install -D -m 0640 "$SRC_KEY"  "$DST_KEY"
+    # Stalwart reads the key as user "stalwart" via %{file:...}%; regrant the
+    # group immediately so every sync leaves it readable (0640 root:stalwart)
+    # and webadmin config reload never fails with EACCES.
+    chgrp stalwart "$DST_KEY"
 
     # 2) TLSA 3 1 1 = SHA-256 of the certificate's SubjectPublicKeyInfo (SPKI),
     #    NOT the whole certificate. Matching type 1 = SHA-256 of the SPKI DER.

servers/hetzner/modules/services/virtualcam-build.patch

@@ -1,29 +0,0 @@
---- a/src/lib/catalog.ts
-+++ b/src/lib/catalog.ts
-@@ -1,3 +1,8 @@
- import { prisma } from "@/lib/db";
-+
-+// During `next build`, Next evaluates generateStaticParams/generateMetadata for
-+// each route, which calls these DB readers. No database exists in the sandboxed
-+// Nix build, so short-circuit them here; the live site fetches real rows/request.
-+const BUILD = process.env.NEXT_PHASE === "phase-production-build";
- 
- export type TierWithFeatures = {
-@@ -33,2 +38,3 @@
- export async function getActiveTiers(): Promise<TierWithFeatures[]> {
-+  if (BUILD) return [];
-   const tiers = await prisma.tier.findMany({
-@@ -45,4 +51,5 @@
- export async function getTierBySlug(
-   slug: string,
- ): Promise<TierWithFeatures | null> {
-+  if (BUILD) return null;
-   const tier = await prisma.tier.findUnique({
-@@ -58,2 +65,3 @@
- export async function getDocPages() {
-+  if (BUILD) return [];
-   return prisma.docPage.findMany({
-@@ -65,2 +73,3 @@
- export async function getDocBySlug(slug: string) {
-+  if (BUILD) return null;
-   return prisma.docPage.findUnique({ where: { slug } });

servers/hetzner/modules/services/virtualcam-checkout.patch

@@ -0,0 +1,32 @@
+--- a/src/app/api/checkout/route.ts
++++ b/src/app/api/checkout/route.ts
+@@ -7,6 +7,8 @@
+ 
+ export const dynamic = "force-dynamic";
+ 
++const BYPASS_PAYMENTS = process.env.BYPASS_PAYMENTS === "true";
++
+ export async function POST(req: Request) {
+   // Each checkout hits the payment provider, so cap order creation per user
+   // and per IP to prevent order spam and provider-API abuse.
+@@ -86,6 +88,20 @@
+     const paid = await prisma.order.findUnique({
+       where: { id: order.id },
+       include: { license: true },
++    });
++    return NextResponse.json({
++      ok: true,
++      orderId: order.id,
++      licenseKey: paid?.license?.key ?? null,
++      tier: tier.slug,
++    });
++  }
++
++  if (BYPASS_PAYMENTS) {
++    await markOrderPaid({ orderId: order.id, providerRef: "bypass-test" });
++    const paid = await prisma.order.findUnique({
++      where: { id: order.id },
++      include: { license: true },
+     });
+     return NextResponse.json({
+       ok: true,

servers/hetzner/modules/services/virtualcam-layout.patch

@@ -1,30 +0,0 @@
---- a/src/app/layout.tsx
-+++ b/src/app/layout.tsx
-@@ -1,19 +1,14 @@
- import type { Metadata } from "next";
--import { Pixelify_Sans } from "next/font/google";
- import "./globals.css";
- import { getCurrentUser } from "@/lib/auth";
- import { getAppUrl } from "@/lib/env";
- import { Providers } from "@/components/providers";
- import { AuthProvider } from "@/contexts/auth-context";
- import { MatrixEffects } from "@/components/matrix-effects";
- import { SiteHeader } from "@/components/site-header";
- import { SiteFooter } from "@/components/site-footer";
--
--const pixelify = Pixelify_Sans({
--  subsets: ["latin"],
--  weight: ["400", "500", "600", "700"],
--  variable: "--font-pixelify",
--  display: "swap",
--});
--
-+// Built fully dynamic so the sandboxed Nix build needs no database and no
-+// external font download; also correct for a DB-driven storefront.
-+export const dynamic = "force-dynamic";
-+
- export const metadata: Metadata = {
-@@ -50,2 +45,2 @@
--    <html lang="en" className={`${pixelify.variable} h-full`} suppressHydrationWarning>
-+    <html lang="en" className="h-full" suppressHydrationWarning>
-       <body className="min-h-full flex flex-col bg-background text-foreground">

servers/hetzner/modules/services/virtualcam.nix

@@ -4,8 +4,8 @@
   unstablePkgs,
   ...
 }: let
-  rev = "0e7135918c251ce6bcc2ead1d3f4cf0075ae2c55";
-  apiRev = "a73f4c4be840444072b0a7f5458438a34b470ea9";
+  rev = "6e351ddc732421eb775eb9843ccd2389d6525edd";
+  apiRev = "390878d126185b21a14479f85c70134a155c61c6";
 
   # Private repositories are fetched over SSH (port 2222). nix-daemon runs as
   # root and uses /root/.ssh (identity materialized by the git-ssh-key unit),
@@ -15,6 +15,12 @@
     rev = rev;
   };
 
+  srcPatched = pkgs.applyPatches {
+    name = "virtualcam-website-patched";
+    src = src;
+    patches = [./virtualcam-checkout.patch];
+  };
+
   apiSrc = builtins.fetchGit {
     url = "ssh://git@git.severijnse.eu:2222/jory/virtualcam-api.git";
     rev = apiRev;
@@ -24,8 +30,8 @@
   app = unstablePkgs.buildNpmPackage {
     pname = "virtualcam-website";
     version = "0.1.0";
-    src = src;
-    npmDepsHash = "sha256-0g98Jh/RwoicjrfiSbfqNo331k3ab8hINjV6dHHN0y4=";
+    src = srcPatched;
+    npmDepsHash = "sha256-GodWQKtOtsLOjjiwzxun+wTPhtvjR2uCV91n+wiZHw4=";
     nodejs = unstablePkgs.nodejs;
 
     buildPhase = ''
@@ -44,6 +50,7 @@
     '';
 
     APP_URL = "https://virtualcam.severijnse.eu";
+    GOOGLE_CLIENT_ID = "754775011707-c699m092tv3icmovhk5qa106v3q6eh7c.apps.googleusercontent.com";
     # Only used to satisfy prisma generate / next build metadata resolution.
     DATABASE_URL = "postgresql://virtualcam@localhost/virtualcam?host=/run/postgresql&schema=public";
     # Use the nixpkgs-bundled Prisma engine so the sandboxed offline build does
@@ -88,7 +95,7 @@
     chmod 0600 "${envFile}"
     ${pkgs.sops}/bin/sops --decrypt --input-type yaml --output-type yaml ${secretsFile} |
       ${pkgs.gnused}/bin/sed -nE \
-        's/^virtualcam_license_signing_key: (.*)/LICENSE_SIGNING_KEY=\1/p; s/^virtualcam_admin_token: (.*)/ADMIN_TOKEN=\1/p' \
+         's/^virtualcam_license_signing_key: (.*)/LICENSE_SIGNING_KEY=\1/p; s/^virtualcam_admin_token: (.*)/ADMIN_TOKEN=\1/p; s/^virtualcam_smtp_pass: (.*)/SMTP_PASS=\1/p; s/^virtualcam_google_OAuth_secret: (.*)/GOOGLE_CLIENT_SECRET=\1/p; s/^shkeeper_api_key: (.*)/SHKEEPER_API_KEY=\1/p' \
         >> "${envFile}"
   '';
 
@@ -214,8 +221,8 @@ in {
 
       virtualcam = {
         description = "Virtualcamera website (Next.js)";
-        after = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
-        requires = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
+        after = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
+        requires = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-seed.service" "virtualcam-secrets.service"];
         wantedBy = ["multi-user.target"];
         path = [unstablePkgs.nodejs];
         serviceConfig = {
@@ -231,12 +238,17 @@ in {
           Environment = [
             "DATABASE_URL=${dbUrl}"
             "APP_URL=https://virtualcam.severijnse.eu"
+            "GOOGLE_CLIENT_ID=754775011707-c699m092tv3icmovhk5qa106v3q6eh7c.apps.googleusercontent.com"
             "PAYMENTS_MODE=shkeeper"
+            "BYPASS_PAYMENTS=false"
+            "SHKEEPER_URL=https://pay.severijnse.eu"
+            "SHKEEPER_CRYPTO=BTC"
+            "SHKEEPER_FIAT=USD"
             "ADMIN_EMAILS=jory@severijnse.eu"
-            "SMTP_HOST=localhost"
+            "SMTP_HOST=mail.severijnse.eu"
             "SMTP_PORT=587"
-            "SMTP_USER=jory@severijnse.eu"
-            "SMTP_FROM=noreply@severijnse.eu"
+            "SMTP_USER=no-reply@severijnse.eu"
+            "SMTP_FROM=no-reply@severijnse.eu"
             "NODE_ENV=production"
             "NEXT_TELEMETRY_DISABLED=1"
             "HOME=/var/lib/virtualcam"
@@ -246,8 +258,8 @@ in {
 
       virtualcam-api = {
         description = "Virtualcamera license API (Go/Fiber)";
-        after = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
-        requires = ["postgresql.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
+        after = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
+        requires = ["postgresql.service" "podman-shkeeper.service" "virtualcam-migrate.service" "virtualcam-secrets.service"];
         wantedBy = ["multi-user.target"];
         serviceConfig = {
           User = "virtualcam";

servers/hetzner/modules/system/opencode.nix

@@ -1,10 +0,0 @@
-{pkgs, ...}: {
-  environment.systemPackages = [
-    (pkgs.writeShellScriptBin "opencode" ''
-      exec /home/admin/.local/bin/opencode "$@"
-    '')
-  ];
-
-  programs.fish.shellInit = ''
-  '';
-}

servers/hetzner/secrets/secrets.yaml

@@ -1,3 +1,4 @@
+shkeeper_api_key: ENC[AES256_GCM,data:kUymtefYJOllK5cxKHhD3HnXO/uo1w==,iv:OH9ZaQJZsFfo00BIl5MlRw1XBzHUmFi1FW54/J6O/8Q=,tag:HQJyuEoz5fsDi7iMq3PEzw==,type:str]
 restic_password: ENC[AES256_GCM,data:OHQlxUpNnTqMQm6A/o3ID/3F91NpVKOrsFYdLqrtI4vp+6TWHR8=,iv:bBd+gfi78lgTaTv0QUWYGQWPsurrzx90VvOzA2d2euA=,tag:qsHSdAHx1auwdZCgn2O5Qw==,type:str]
 b2_key_id: ENC[AES256_GCM,data:4B9rvg06baH6aNiT,iv:Qk1ToF3lMYLTrZdzpfaoGVzdiKYs492w9fzn4/TbNfQ=,tag:djxfqwgNZmRGqvteTX9G3w==,type:str]
 b2_application_key: ENC[AES256_GCM,data:xGAbBqx+6ErK7sy3FR0yza6mJU5oc5uQQGcwKtcPp1Ef4XEVd8do/wN7,iv:CojcoykDRBfvN8yqMMrPJq9mtAwxyswqXXVBKPupxDs=,tag:BkfFHAGJvtCDUpFoxXVP3Q==,type:str]
@@ -6,6 +7,8 @@ wg_admin_password: ENC[AES256_GCM,data:X/+YJoNoFFU7P/2HCpTI,iv:FXDS3xuFB9jxjpQhs
 virtualcam_license_signing_key: ENC[AES256_GCM,data:zk8YDER9d9mBOdXgEKKC6dubSMfz6UMMb758ZC9SvjkYIv+R2EVd+CJwL93MCs0l62lUDxoBDCgTLbKM55kcaThkkqfgZCYPrndXevYZ7kquNQEMkoYg/pyGAWhAQ9trvT5QXfBES9UDdYi6J9taCVKCofutiMEdoJM8olgUwKsP4i+fbt3idYkdWNGelastPnvTuUK/M3/dOKR5eheiy5oIntG+sECOz/2eKHJ8cU3ja18uAEwcpd9VojuOa0fvfui8AujQcMZNSMv8VsLGnoR854iX3g1MvHkTBSzv3qaDzRC54MueXYR4WOzm0o+yv6Otmw1n9trWtk+WLUimmiKxDdW6nSFaddeisWzy47+QYzrSsd7iovNYEIHnW4kpnT5W2g5lwaJwA0J0pBOqvV9gziv8oBNeEbLurBxi78KwHBqG,iv:svx+hC4tS26xqI05+yo6N2d3uHzO7ULM0e76dMbShjs=,tag:EJ9fjHEi1wUhLkxy8sexww==,type:str]
 virtualcam_admin_token: ENC[AES256_GCM,data:JC5zSFBCoEapxv/Kcs4VXk3HAkiqwXVrCv1rlqFsXVx3LGspnQdzjOIahlM=,iv:GRVfMNGkwC3xXR00ww29rFbhTrpuViad+H05yDhR9BA=,tag:OBnxBD/8KU4mn17FFTcdng==,type:str]
 git_ssh_key_b64: ENC[AES256_GCM,data:LaUcR005w9iCd/DifDhzLM7tU4AuTiFM802gMU1HMtvFN2iSffucaFJWxpKv2zFxDxz0QpA3XwM1M5yJC4K3YFMxPkPXcWLYN4yHcc1yXuoB3SseQXjcJnPMqZvJfVYhHep+6CVVlVDQs1Y+miBvbAhrvCEblwUu5LhD7yvoJLwTnEpT15WGDvqezzYKzNSyQGNRa8KPjsD/7gJN3RyUPzKo6GsrIp8d8eOTuCZwEcJVDKE0ACXs4V6cd+akqIWZkNPyilnMTuK5cqg2YuqSpwu9NncKED6SjmPiozPKKgTVaWhNCRJUTBfIw85fdXizvq7PNbNRxxBsscAWd0ixXiLqg1RTipfv85wL4Nru3r3qcQGAWmOpTHKW5fA9kR4ygdNxEVkOgQ70m9U0wev+vdIJUoRYj09Ikobej370NKppOP6+oeEDE7NpkFH7tzRk1oHtwoc1dwP9Kq1T8XdxttCKFyb1pAJ6gMdMThCfJlC9/Uu/WYbYVu5bCj6BAmrdrmhbPrsEOylBeC9j9RhyRrFEqLd8AZ01Zg8oKcj9WtCWEMf9Y4biv2xE43JbBGRvLAx3sH9MRplu2hN2W/bRxa/EPj9GHgbUjScDBkm/QVbMeV+QOgi09sCHDTeSX3BHdZ8ZLNjuNUwYnWZq4lPix0iYS5MufiixNdQmncPeHq0UuBtVifEl2Jmjq1COLyeAQe47AKQ5zDjT91xtBRZVZEVk0kGBYHtBEJ9ezPwckSw=,iv:cbIpwwhmyJVvvTMB95YBzt+RQLOji6yJh+SpcJc3Q1Y=,tag:t8WxU1IPDfkB8muTdBLUhg==,type:str]
+virtualcam_smtp_pass: ENC[AES256_GCM,data:D5kDTQP9/YV5BjAWeTn0G2zKe+s=,iv:7dZJprrs0S2ECtbDQuvhUWsU0Xzpld5/XZqdbyefg0o=,tag:uWe811KAisBJ5M1EhJjrcQ==,type:str]
+virtualcam_google_OAuth_secret: ENC[AES256_GCM,data:kPJIP4iH2vFo+VRojvhjzrQhDWidFq7kjC7ZaFcwJ7dNHlo=,iv:qKWClwSIRPU2Xz50bGYqU+eMR/iFxqnaKkb18eL3aMU=,tag:78jGQgL1myDBMiZP8GZ80g==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -39,8 +42,8 @@ sops:
             MHJrVVpDYWdJNmxtUkozSzR4Nmt3R28KrhYi830HUFAPfg8WvPad7BAuNe1mYOWt
             WEFIquuX/H/N+y/7uQcBDbvnBzyropE1hW8aNrxSKMeawvQZWNXkZA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-15T14:55:25Z"
-    mac: ENC[AES256_GCM,data:M8Br3hpJdAtaIku07AnnnR/A3zoZLEE+njE+BX7XBaaTvjcLMs6TKUl79EMlpXLQIXR2nchrf7dE3qlqxnbotw0WjhCOW7KHJskv/nTq5ZO7yQW817whAF7clDmQVhuA0teOGqwY3MKzZ5+2BUmsBLREkH3ZIQ7x/IgWyL/WUJE=,iv:960gzieWk4S1Wily3Kh32lKPbteFToXT5g8/Dp/pDOU=,tag:vDi1L5N/G/Wcyqtl18oG7w==,type:str]
+    lastmodified: "2026-08-19T10:30:19Z"
+    mac: ENC[AES256_GCM,data:JOWFguh2iE4+OyU00l+2gn7GWv2yej37OLLhVVO6Yb7Kv4vgk4NhZKJLqopteQK1o6dPeWbeIydHG1Qmef0Sm0NmCyi7i6BCTplVizf/puaGjpkU5YAs90+HiEa9cKlNQ5brfNH3YecJeN9dK+d7o8bMO/xPv3raqFMBBoVtgu4=,iv:A7kCOzyWwiVaXKVMKXQ4j86zz2f65wAjOtaVEuLa81I=,tag:bcbPsmRRFRHX7qi4WdYb0g==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

system/nix/default.nix

@@ -19,6 +19,16 @@
   in {
     package = pkgs.lix;
 
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 14d";
+    };
+
+    optimise = {
+      automatic = true;
+    };
+
     # pin the registry to avoid downloading and evaling a new nixpkgs version every time
     registry = lib.mapAttrs (_: v: {flake = v;}) flakeInputs;