feat(hetzner): full migration of docker-mailserver to stalwart

result

@@ -1 +1 @@
-/nix/store/d8ysys9mxibkvpiskrvsdiybhwkdnj3d-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951
+/nix/store/3rwh3r83dmni3kkfmfnrh36lv0pb2zxn-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951

servers/hetzner/hosts/hetzner/default.nix

@@ -12,7 +12,6 @@
     ../../modules/services/wireguard.nix
     ../../modules/services/fail2ban.nix
     ../../modules/services/vaultwarden.nix
-    ../../modules/services/mailserver.nix
     ../../modules/services/rustdesk.nix
     ../../modules/services/wrxproxy.nix
     ../../modules/services/watchtower.nix

servers/hetzner/modules/services/backup.nix

@@ -57,12 +57,13 @@ in {
       "/var/lib/gitea"
       "/var/lib/caddy"
       "/var/lib/virtualcam"
+      "/var/lib/stalwart"
+      "/var/lib/bulwark"
       "/var/lib/coredns"
       "/etc/nixos"
     ];
     exclude = [
       "/home/admin/backups"
-      "/home/admin/dms/mail-logs"
       "/home/admin/.opencode"
       "/home/admin/.local"
       "/home/admin/.npm"

servers/hetzner/modules/services/caddy.nix

@@ -161,8 +161,23 @@ in {
         extraConfig = ''
           import security_headers
           ${antiScrape}
-          # Bulwark webmail (JMAP client for Stalwart), running on host port 3002.
-          reverse_proxy 127.0.0.1:3002
+          # Stalwart JMAP API (http-management listener on 8080): required so the
+          # browser can reach /.well-known/jmap, /jmap/session and the JMAP
+          # upload/download/eventsource/ws paths at the public origin.
+          handle /.well-known/jmap {
+            reverse_proxy 127.0.0.1:8080
+          }
+          # NOTE: use handle (not handle_path) so the /jmap prefix is preserved.
+          handle /jmap {
+            reverse_proxy 127.0.0.1:8080
+          }
+          handle /jmap/* {
+            reverse_proxy 127.0.0.1:8080
+          }
+          handle {
+            # Bulwark webmail (JMAP client for Stalwart), running on host port 3002.
+            reverse_proxy 127.0.0.1:3002
+          }
           encode zstd gzip
         '';
       };

servers/hetzner/modules/services/coredns.nix

@@ -27,7 +27,7 @@
 
     mail._domainkey.severijnse.eu. 300 IN TXT (
       "v=DKIM1; k=rsa; "
-      "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8/"
+      "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8K/"
       "JaoTC8OURk365+/aBSmWXUCCBphCx43QWfC9h8GMQ6PUIqawkz5CcGPT7X7hPdwHQcd5Vn3CqmABptxdwshdkBjZs"
       "oi79BOo9ZrQSTY7iiLcOP7hVVC9Ad+ydlZ4MWGfy5BxgyTGrrtuSuLcM219oqdovIvr2EtXs8AMx5fyXplKE3R/"
       "YlwF2Jcy50Gmb5y/E9pOaFjVv8HXUmKsvuhA2b8K+rt0WVHNc3dvbgZUl8bGAQIDAQAB"

servers/hetzner/modules/services/fail2ban.nix

@@ -4,7 +4,6 @@
     image = "crazymax/fail2ban:latest";
     autoStart = true;
     volumes = [
-      "/home/admin/dms/mail-logs:/var/log/mail:ro"
       "/home/admin/fail2ban/data:/data:Z"
     ];
     environment = {
@@ -15,8 +14,4 @@
       "--cap-add=NET_ADMIN"
     ];
   };
-
-  systemd.tmpfiles.rules = [
-    "d /var/log/mail 0755 root root -"
-  ];
 }

servers/hetzner/modules/services/stalwart.nix

@@ -12,7 +12,7 @@
   # Root-only runtime file holding the fallback-admin password hash (0600 root).
   adminHashFile = "/var/lib/stalwart/.admin-hash";
 
-  # Materialize the fallback-admin password hash from sops into a root-only file.
+# Materialize the fallback-admin password hash from sops into a root-only file.
   writeAdminHash = pkgs.writeShellScript "stalwart-write-admin-hash" ''
     set -euo pipefail
     install -d -o root -g root -m 0755 "$(dirname ${adminHashFile})"
@@ -22,6 +22,18 @@
       | tr -d '\n' > "${adminHashFile}"
     chmod 0600 "${adminHashFile}"
   '';
+
+  # The outbound DKIM key is the existing opendkim key from docker-mailserver
+  # (selector "mail", domain severijnse.eu, PKCS#8 RSA 2048). Install it into the
+  # stalwart-owned data dir so the server can read it and it is covered by the
+  # /var/lib/stalwart backup.
+  dkimKeyDir = "/var/lib/stalwart/dkim/severijnse.eu";
+  dkimKeySrc = "/home/admin/dms/config/opendkim/keys/severijnse.eu/mail.private";
+  writeDkimKey = pkgs.writeShellScript "stalwart-write-dkim-key" ''
+    set -euo pipefail
+    install -d -o stalwart -g stalwart -m 0750 ${dkimKeyDir}
+    install -o stalwart -g stalwart -m 0640 ${dkimKeySrc} ${dkimKeyDir}/mail.private
+  '';
 in {
   # The hetzner host is built with nixos-24.05, which ships its own
   # `services.stalwart-mail` module (for the old 0.8.x package). We want the
@@ -41,16 +53,27 @@ in {
     stateVersion = "26.05";
     package = unstablePkgs.stalwart;
 
-    # Temporary internal listeners while docker-mailserver still owns 25/143/465/587/993.
+    # Public listeners. The host firewall (networking.nix) already allows
+    # 25/143/465/587/993, so openFirewall stays off: the module would also open
+    # every other listener port incl. the 8080 webadmin.
     openFirewall = false;
 
     settings = {
       # EHLO / hostname for the server (docs server.hostname).
       server.hostname = "mail.severijnse.eu";
 
+      # Public origin the JMAP/webadmin API is served from, advertised in the
+      # JMAP session (docs server/core/network.md "http.url"). Value must be a
+      # JScript expression, hence the single-quoted string literal. Without this
+      # Stalwart advertises http://mail.severijnse.eu:8080, which the browser
+      # refuses to fetch and breaks Bulwark webmail.
+      http.url = "'https://mail.severijnse.eu'";
+
       certificate."mail-severijnse-eu" = {
         cert = "%{file:${certDir}/mail.severijnse.eu.crt}%";
         private-key = "%{file:${certDir}/mail.severijnse.eu.key}%";
+        # Docs server/tls/certificates.md: used when the client sends no SNI.
+        default = true;
       };
 
       server.tls = {
@@ -59,18 +82,32 @@ in {
         implicit = false;
       };
 
-      # Temporary internal listeners (docs server/listener.md + protocol, tls.implicit override).
+      # Public listeners (docs server/listener.md + protocol, tls.implicit override).
+      # Bind "[::]:port" for dual-stack IPv4+IPv6 (docs: "to bind a listener to
+      # all interfaces"); listing both 0.0.0.0 and [::] makes the [::] bind fail
+      # with EADDRINUSE on kernels with net.ipv6.bindv6only=0.
+      # 143/587 use STARTTLS (server.tls.implicit=false default), 993/465 the
+      # implicit-TLS variants, 25 the plain (STARTTLS) MX port.
       server.listener = {
         "imap" = {
-          bind = ["127.0.0.1:1143"];
+          bind = ["[::]:143"];
           protocol = "imap";
         };
+        "imaps" = {
+          bind = ["[::]:993"];
+          protocol = "imap";
+          tls.implicit = true;
+        };
+        "smtp" = {
+          bind = ["[::]:25"];
+          protocol = "smtp";
+        };
         "smtp-submission" = {
-          bind = ["127.0.0.1:1587"];
+          bind = ["[::]:587"];
           protocol = "smtp";
         };
         "smtp-submissions" = {
-          bind = ["127.0.0.1:1465"];
+          bind = ["[::]:465"];
           protocol = "smtp";
           tls.implicit = true;
         };
@@ -80,10 +117,29 @@ in {
         };
       };
 
-      # Auth per inbound/auth.md: not required on the plain SMTP listener (port 25),
-      # required everywhere else (IMAP + submission). Directory is the module default "internal".
-      session.auth.mechanisms = "[plain]";
-      session.auth.directory = "'internal'";
+      # Auth per docs mta/inbound/auth.md (AUTH stage): authentication is
+      # disabled on the plain SMTP listener (port 25) and required everywhere
+      # else (IMAP + submission). Only offer PLAIN/LOGIN over TLS, so clear-text
+      # listeners (143/587 pre-STARTTLS) advertise no SASL mechanisms. This
+      # mirrors the code defaults (crates/common/src/config/smtp/session.rs).
+      # The JMAP/webadmin "http-management" listener (bind 127.0.0.1:8080) is
+      # plain HTTP and reachable only from localhost, where bulwark connects;
+      # its basic-auth uses the "plain" mechanism, so it must be exempted from
+      # the is_tls gate or webmail logins fail with "Authentication not allowed".
+      session.auth.mechanisms = [
+        {
+          "if" = "local_port != 25 && (is_tls || listener == 'http-management')";
+          "then" = "[plain, login]";
+        }
+        {"else" = false;}
+      ];
+      session.auth.directory = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = "'internal'";
+        }
+        {"else" = false;}
+      ];
       session.auth.require = [
         {
           "if" = "listener != 'smtp'";
@@ -92,6 +148,43 @@ in {
         {"else" = false;}
       ];
 
+      # Outbound DKIM signing (docs mta/authentication/dkim/sign): sign with the
+      # "mail" signature on everything submitted via non-25 listeners; do not sign
+      # inbound mail received on the plain "smtp" listener.
+      auth.dkim.sign = [
+        {
+          "if" = "listener != 'smtp'";
+          "then" = "['mail']";
+        }
+        {"else" = false;}
+      ];
+
+      # ARC sealing uses the same "mail" signature (docs mta/authentication/arc).
+      # The code default ('rsa-' + report.domain) would reference a signature
+      # name that does not exist and log "ARC sealer not found".
+      auth.arc.seal = "'mail'";
+
+      # Every *downstream* sign rule defaults to signing with
+      # ['rsa-<report.domain>', 'ed25519-<report.domain>'] (queue.rs/report.rs),
+      # names that do not exist here and log "DKIM signer not found" on DSNs and
+      # reports. Point them all at the real "mail" signature.
+      report.dsn.sign = "['mail']";
+      report.spf.sign = "['mail']";
+      report.dmarc.sign = "['mail']";
+      report.dmarc.aggregate.sign = "['mail']";
+      report.tls.aggregate.sign = "['mail']";
+
+      # Reuse the existing opendkim key (selector mail) so no DNS change is needed.
+      signature.mail = {
+        private-key = "%{file:/var/lib/stalwart/dkim/severijnse.eu/mail.private}%";
+        domain = "severijnse.eu";
+        selector = "mail";
+        headers = ["From" "To" "Date" "Subject" "Message-Id"];
+        algorithm = "rsa-sha256";
+        canonicalization = "relaxed/relaxed";
+        set-body-length = false;
+      };
+
       # Fallback admin (auth/authorization/administrator.md): bootstrap admin with
       # every permission, used to create the internal-directory accounts via the
       # management REST API / CLI. Secret is a SHA-512-crypt hash, injected via
@@ -102,8 +195,10 @@ in {
         secret = "%{file:/run/credentials/stalwart.service/stalwart-admin}%";
       };
 
-      # Route docs routing: /strategy.md + /routing.md:
-      # local domains → local store, everything else → MX. local/mx are built-in.
+      # Route docs mta/outbound/routing.md + strategy.md: local domains → local
+      # store, everything else → MX. The strategy names are defined explicitly
+      # (docs define queue.route.mx/local; the built-in fallback in
+      # core.rs:get_route_or_default only kicks in for undeclared names).
       queue.strategy.route = [
         {
           "if" = "is_local_domain('', rcpt_domain)";
@@ -111,6 +206,13 @@ in {
         }
         {"else" = "'mx'";}
       ];
+      queue.route."mx" = {
+        type = "mx";
+        ip-lookup = "ipv4_then_ipv6";
+      };
+      queue.route."local" = {
+        type = "local";
+      };
     };
   };
 
@@ -118,8 +220,8 @@ in {
   # ProtectSystem=strict). The TLS key tlsa-updater installs is 0640 root:root;
   # regrant it to the stalwart group after every cert sync so stalwart can serve TLS.
   systemd.services.stalwart = {
-    after = ["tlsa-update.service" "stalwart-admin-secret.service"];
-    requires = ["tlsa-update.service" "stalwart-admin-secret.service"];
+    after = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"];
+    requires = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"];
   };
 
   # Make the management CLI available for account creation and maildir import
@@ -142,6 +244,17 @@ in {
     };
   };
 
+  # Install the outbound DKIM key into the stalwart data dir before the service starts.
+  systemd.services.stalwart-dkim = {
+    description = "Install Stalwart outbound DKIM key";
+    wantedBy = ["multi-user.target"];
+    before = ["stalwart.service"];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${writeDkimKey}";
+    };
+  };
+
   # LoadCredential: expose the materialized hash to stalwart only at
   # /run/credentials/stalwart.service/stalwart-admin (see `credentials` option
   # in the upstream module; the value is the source path on disk).
@@ -172,7 +285,11 @@ in {
       "/var/lib/bulwark:/app/data:Z"
     ];
     environment = {
-      JMAP_SERVER_URL = "http://127.0.0.1:8080";
+      # JMAP_SERVER_URL is the public origin (the browser uses it verbatim for
+      # /.well-known/jmap + the session apiUrl). HOSTNAME stays 127.0.0.1 so
+      # Next.js binds to loopback (caddy reverse_proxy's 127.0.0.1:3002); binding
+      # to the public hostname made the container unreachable for caddy (502).
+      JMAP_SERVER_URL = "https://mail.severijnse.eu";
       HOSTNAME = "127.0.0.1";
       PORT = "3002";
     };

servers/hetzner/modules/services/tlsa-updater.nix

@@ -1,11 +1,15 @@
 {pkgs, ...}: let
   # Caddy's canonical certificate storage (XDG data dir). Renewals land here,
-  # owned caddy:caddy 0600 — the mail server's non-root Postfix/Dovecot cannot
-  # read it directly, so we copy it into a world-readable distribution dir.
+  # owned caddy:caddy 0600 — stalwart cannot read it directly, so we copy it
+  # into a distribution dir stalwart's service (user "stalwart") can reach.
   caddyCertDir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
-  # World-readable distribution dir mounted (RO) into the mail server container.
+  # Distribution dir read by stalwart (cert 0644, private key regranted to the
+  # "stalwart" group by systemd.services.stalwart-cert-perm).
   distCertDir = "/var/lib/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu";
   zoneFile = "/var/lib/coredns/zones/severijnse.eu.db";
+  # Records the SPKI hash applied at the last restart of stalwart, so cert
+  # renewals trigger exactly one restart and unchanged certs never do.
+  stateFile = "/var/lib/tlsa-update/.last-spki";
 
   syncScript = pkgs.writeShellScript "tlsa-update" ''
     set -euo pipefail
@@ -22,11 +26,11 @@
       exit 0
     fi
 
-    # 1) Propagate Caddy's renewed certificate into the distribution dir the
-    #    mail server mounts. Caddy stores certs 0600 caddy:caddy. The cert is
-    #    world-readable (Postfix/Dovecot read it as root before dropping
-    #    privileges); the private key is restricted to root (0640) so it is not
-    #    exposed to other local users.
+    # 1) Propagate Caddy's renewed certificate into the distribution dir stalwart
+    #    reads. Caddy stores certs 0600 caddy:caddy. The cert is world-readable;
+    #    the private key is restricted to root (0640) and stalwart-cert-perm
+    #    regrants it to the "stalwart" group so it is not exposed to other local
+    #    users.
     install -D -m 0644 "$SRC_CERT" "$DST_CERT"
     install -D -m 0640 "$SRC_KEY"  "$DST_KEY"
 
@@ -47,8 +51,24 @@
 
     # 4) Reload services so the changes take effect immediately.
     systemctl reload coredns.service || true
-    podman exec mailserver postfix reload || true
-    podman exec mailserver dovecot reload || true
+    # Stalwart reads its TLS certs ($certDir) via %{file:...}% placeholders at
+    # startup only; there is no signal-based reload (management API reload needs
+    # admin credentials we must not store). Restart it, but only when the cert
+    # actually changed (SPKI hash differs from the last applied run), so the
+    # daily timer and inotify events for unchanged certs do not drop connections.
+    # NOTE: stalwart.service `requires` this unit, so the restart must be issued
+    # with --no-block (async): a synchronous restart waits for stalwart to come
+    # back up, which in turn waits for this unit to finish — a deadlock. The state
+    # file is updated BEFORE the restart so the tlsa-update run that stalwart's
+    # required-activation re-triggers sees a matching hash and exits immediately.
+    if [ "$(cat "${stateFile}" 2>/dev/null || true)" != "$HEX" ]; then
+      printf '%s\n' "$HEX" > "${stateFile}"
+      # Restarting stalwart re-runs its Requires=tlsa-update dependency; guard
+      # with is-active so a boot-time run never races stalwart's initial start.
+      if systemctl is-active --quiet stalwart.service 2>/dev/null; then
+        systemctl --no-block restart stalwart.service || true
+      fi
+    fi
 
     echo "tlsa-update: TLSA set to $HEX"
   '';
@@ -57,6 +77,7 @@ in {
     # Ensure the distribution dir exists (Caddy does not write here).
     tmpfiles.rules = [
       "d ${distCertDir} 0755 root root - -"
+      "d /var/lib/tlsa-update 0755 root root - -"
     ];
 
     services.tlsa-update = {
@@ -64,7 +85,7 @@ in {
       after = ["caddy.service" "coredns.service"];
       partOf = ["coredns.service"];
       wantedBy = ["multi-user.target"];
-      path = with pkgs; [openssl coreutils gnused podman systemd];
+      path = with pkgs; [openssl coreutils gnused systemd];
       serviceConfig = {
         Type = "oneshot";
         ExecStart = "${syncScript}";