From 31de8eccb00bd2f6f6f476ca677e41c7c6dac01a Mon Sep 17 00:00:00 2001 From: Jory Severijnse Date: Sat, 15 Aug 2026 15:59:27 +0200 Subject: [PATCH] feat(hetzner): full migration of docker-mailserver to stalwart --- result | 2 +- servers/hetzner/hosts/hetzner/default.nix | 1 - servers/hetzner/modules/services/backup.nix | 3 +- servers/hetzner/modules/services/caddy.nix | 19 ++- servers/hetzner/modules/services/coredns.nix | 2 +- servers/hetzner/modules/services/fail2ban.nix | 5 - servers/hetzner/modules/services/stalwart.nix | 147 ++++++++++++++++-- .../hetzner/modules/services/tlsa-updater.nix | 43 +++-- 8 files changed, 185 insertions(+), 37 deletions(-) diff --git a/result b/result index 382bdf6..a41c615 120000 --- a/result +++ b/result @@ -1 +1 @@ -/nix/store/d8ysys9mxibkvpiskrvsdiybhwkdnj3d-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951 \ No newline at end of file +/nix/store/3rwh3r83dmni3kkfmfnrh36lv0pb2zxn-nixos-system-debian-4gb-fsn1-1-24.05.20241230.b134951 \ No newline at end of file diff --git a/servers/hetzner/hosts/hetzner/default.nix b/servers/hetzner/hosts/hetzner/default.nix index 4d0bc4b..1e95e01 100644 --- a/servers/hetzner/hosts/hetzner/default.nix +++ b/servers/hetzner/hosts/hetzner/default.nix @@ -12,7 +12,6 @@ ../../modules/services/wireguard.nix ../../modules/services/fail2ban.nix ../../modules/services/vaultwarden.nix - ../../modules/services/mailserver.nix ../../modules/services/rustdesk.nix ../../modules/services/wrxproxy.nix ../../modules/services/watchtower.nix diff --git a/servers/hetzner/modules/services/backup.nix b/servers/hetzner/modules/services/backup.nix index 192da97..ce079c4 100644 --- a/servers/hetzner/modules/services/backup.nix +++ b/servers/hetzner/modules/services/backup.nix @@ -57,12 +57,13 @@ in { "/var/lib/gitea" "/var/lib/caddy" "/var/lib/virtualcam" + "/var/lib/stalwart" + "/var/lib/bulwark" "/var/lib/coredns" "/etc/nixos" ]; exclude = [ "/home/admin/backups" - "/home/admin/dms/mail-logs" "/home/admin/.opencode" "/home/admin/.local" "/home/admin/.npm" diff --git a/servers/hetzner/modules/services/caddy.nix b/servers/hetzner/modules/services/caddy.nix index 442df54..53d10bb 100644 --- a/servers/hetzner/modules/services/caddy.nix +++ b/servers/hetzner/modules/services/caddy.nix @@ -161,8 +161,23 @@ in { extraConfig = '' import security_headers ${antiScrape} - # Bulwark webmail (JMAP client for Stalwart), running on host port 3002. - reverse_proxy 127.0.0.1:3002 + # Stalwart JMAP API (http-management listener on 8080): required so the + # browser can reach /.well-known/jmap, /jmap/session and the JMAP + # upload/download/eventsource/ws paths at the public origin. + handle /.well-known/jmap { + reverse_proxy 127.0.0.1:8080 + } + # NOTE: use handle (not handle_path) so the /jmap prefix is preserved. + handle /jmap { + reverse_proxy 127.0.0.1:8080 + } + handle /jmap/* { + reverse_proxy 127.0.0.1:8080 + } + handle { + # Bulwark webmail (JMAP client for Stalwart), running on host port 3002. + reverse_proxy 127.0.0.1:3002 + } encode zstd gzip ''; }; diff --git a/servers/hetzner/modules/services/coredns.nix b/servers/hetzner/modules/services/coredns.nix index 285f38e..6ceb4b2 100644 --- a/servers/hetzner/modules/services/coredns.nix +++ b/servers/hetzner/modules/services/coredns.nix @@ -27,7 +27,7 @@ mail._domainkey.severijnse.eu. 300 IN TXT ( "v=DKIM1; k=rsa; " - "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8/" + "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAskpG7m4kninxRE4JF5KzpnBhLbOlGJL2RO/iDfzdz6sHEvxe78q9c5UnZ0OQddOSuQo4q48dJkXR/XzqY7Ak109lhoAx+Kr1neYsi8K/" "JaoTC8OURk365+/aBSmWXUCCBphCx43QWfC9h8GMQ6PUIqawkz5CcGPT7X7hPdwHQcd5Vn3CqmABptxdwshdkBjZs" "oi79BOo9ZrQSTY7iiLcOP7hVVC9Ad+ydlZ4MWGfy5BxgyTGrrtuSuLcM219oqdovIvr2EtXs8AMx5fyXplKE3R/" "YlwF2Jcy50Gmb5y/E9pOaFjVv8HXUmKsvuhA2b8K+rt0WVHNc3dvbgZUl8bGAQIDAQAB" diff --git a/servers/hetzner/modules/services/fail2ban.nix b/servers/hetzner/modules/services/fail2ban.nix index 079700d..d6e75a9 100644 --- a/servers/hetzner/modules/services/fail2ban.nix +++ b/servers/hetzner/modules/services/fail2ban.nix @@ -4,7 +4,6 @@ image = "crazymax/fail2ban:latest"; autoStart = true; volumes = [ - "/home/admin/dms/mail-logs:/var/log/mail:ro" "/home/admin/fail2ban/data:/data:Z" ]; environment = { @@ -15,8 +14,4 @@ "--cap-add=NET_ADMIN" ]; }; - - systemd.tmpfiles.rules = [ - "d /var/log/mail 0755 root root -" - ]; } diff --git a/servers/hetzner/modules/services/stalwart.nix b/servers/hetzner/modules/services/stalwart.nix index d4353ca..2e6d11b 100644 --- a/servers/hetzner/modules/services/stalwart.nix +++ b/servers/hetzner/modules/services/stalwart.nix @@ -12,7 +12,7 @@ # Root-only runtime file holding the fallback-admin password hash (0600 root). adminHashFile = "/var/lib/stalwart/.admin-hash"; - # Materialize the fallback-admin password hash from sops into a root-only file. +# Materialize the fallback-admin password hash from sops into a root-only file. writeAdminHash = pkgs.writeShellScript "stalwart-write-admin-hash" '' set -euo pipefail install -d -o root -g root -m 0755 "$(dirname ${adminHashFile})" @@ -22,6 +22,18 @@ | tr -d '\n' > "${adminHashFile}" chmod 0600 "${adminHashFile}" ''; + + # The outbound DKIM key is the existing opendkim key from docker-mailserver + # (selector "mail", domain severijnse.eu, PKCS#8 RSA 2048). Install it into the + # stalwart-owned data dir so the server can read it and it is covered by the + # /var/lib/stalwart backup. + dkimKeyDir = "/var/lib/stalwart/dkim/severijnse.eu"; + dkimKeySrc = "/home/admin/dms/config/opendkim/keys/severijnse.eu/mail.private"; + writeDkimKey = pkgs.writeShellScript "stalwart-write-dkim-key" '' + set -euo pipefail + install -d -o stalwart -g stalwart -m 0750 ${dkimKeyDir} + install -o stalwart -g stalwart -m 0640 ${dkimKeySrc} ${dkimKeyDir}/mail.private + ''; in { # The hetzner host is built with nixos-24.05, which ships its own # `services.stalwart-mail` module (for the old 0.8.x package). We want the @@ -41,16 +53,27 @@ in { stateVersion = "26.05"; package = unstablePkgs.stalwart; - # Temporary internal listeners while docker-mailserver still owns 25/143/465/587/993. + # Public listeners. The host firewall (networking.nix) already allows + # 25/143/465/587/993, so openFirewall stays off: the module would also open + # every other listener port incl. the 8080 webadmin. openFirewall = false; settings = { # EHLO / hostname for the server (docs server.hostname). server.hostname = "mail.severijnse.eu"; + # Public origin the JMAP/webadmin API is served from, advertised in the + # JMAP session (docs server/core/network.md "http.url"). Value must be a + # JScript expression, hence the single-quoted string literal. Without this + # Stalwart advertises http://mail.severijnse.eu:8080, which the browser + # refuses to fetch and breaks Bulwark webmail. + http.url = "'https://mail.severijnse.eu'"; + certificate."mail-severijnse-eu" = { cert = "%{file:${certDir}/mail.severijnse.eu.crt}%"; private-key = "%{file:${certDir}/mail.severijnse.eu.key}%"; + # Docs server/tls/certificates.md: used when the client sends no SNI. + default = true; }; server.tls = { @@ -59,18 +82,32 @@ in { implicit = false; }; - # Temporary internal listeners (docs server/listener.md + protocol, tls.implicit override). + # Public listeners (docs server/listener.md + protocol, tls.implicit override). + # Bind "[::]:port" for dual-stack IPv4+IPv6 (docs: "to bind a listener to + # all interfaces"); listing both 0.0.0.0 and [::] makes the [::] bind fail + # with EADDRINUSE on kernels with net.ipv6.bindv6only=0. + # 143/587 use STARTTLS (server.tls.implicit=false default), 993/465 the + # implicit-TLS variants, 25 the plain (STARTTLS) MX port. server.listener = { "imap" = { - bind = ["127.0.0.1:1143"]; + bind = ["[::]:143"]; protocol = "imap"; }; + "imaps" = { + bind = ["[::]:993"]; + protocol = "imap"; + tls.implicit = true; + }; + "smtp" = { + bind = ["[::]:25"]; + protocol = "smtp"; + }; "smtp-submission" = { - bind = ["127.0.0.1:1587"]; + bind = ["[::]:587"]; protocol = "smtp"; }; "smtp-submissions" = { - bind = ["127.0.0.1:1465"]; + bind = ["[::]:465"]; protocol = "smtp"; tls.implicit = true; }; @@ -80,10 +117,29 @@ in { }; }; - # Auth per inbound/auth.md: not required on the plain SMTP listener (port 25), - # required everywhere else (IMAP + submission). Directory is the module default "internal". - session.auth.mechanisms = "[plain]"; - session.auth.directory = "'internal'"; + # Auth per docs mta/inbound/auth.md (AUTH stage): authentication is + # disabled on the plain SMTP listener (port 25) and required everywhere + # else (IMAP + submission). Only offer PLAIN/LOGIN over TLS, so clear-text + # listeners (143/587 pre-STARTTLS) advertise no SASL mechanisms. This + # mirrors the code defaults (crates/common/src/config/smtp/session.rs). + # The JMAP/webadmin "http-management" listener (bind 127.0.0.1:8080) is + # plain HTTP and reachable only from localhost, where bulwark connects; + # its basic-auth uses the "plain" mechanism, so it must be exempted from + # the is_tls gate or webmail logins fail with "Authentication not allowed". + session.auth.mechanisms = [ + { + "if" = "local_port != 25 && (is_tls || listener == 'http-management')"; + "then" = "[plain, login]"; + } + {"else" = false;} + ]; + session.auth.directory = [ + { + "if" = "listener != 'smtp'"; + "then" = "'internal'"; + } + {"else" = false;} + ]; session.auth.require = [ { "if" = "listener != 'smtp'"; @@ -92,6 +148,43 @@ in { {"else" = false;} ]; + # Outbound DKIM signing (docs mta/authentication/dkim/sign): sign with the + # "mail" signature on everything submitted via non-25 listeners; do not sign + # inbound mail received on the plain "smtp" listener. + auth.dkim.sign = [ + { + "if" = "listener != 'smtp'"; + "then" = "['mail']"; + } + {"else" = false;} + ]; + + # ARC sealing uses the same "mail" signature (docs mta/authentication/arc). + # The code default ('rsa-' + report.domain) would reference a signature + # name that does not exist and log "ARC sealer not found". + auth.arc.seal = "'mail'"; + + # Every *downstream* sign rule defaults to signing with + # ['rsa-', 'ed25519-'] (queue.rs/report.rs), + # names that do not exist here and log "DKIM signer not found" on DSNs and + # reports. Point them all at the real "mail" signature. + report.dsn.sign = "['mail']"; + report.spf.sign = "['mail']"; + report.dmarc.sign = "['mail']"; + report.dmarc.aggregate.sign = "['mail']"; + report.tls.aggregate.sign = "['mail']"; + + # Reuse the existing opendkim key (selector mail) so no DNS change is needed. + signature.mail = { + private-key = "%{file:/var/lib/stalwart/dkim/severijnse.eu/mail.private}%"; + domain = "severijnse.eu"; + selector = "mail"; + headers = ["From" "To" "Date" "Subject" "Message-Id"]; + algorithm = "rsa-sha256"; + canonicalization = "relaxed/relaxed"; + set-body-length = false; + }; + # Fallback admin (auth/authorization/administrator.md): bootstrap admin with # every permission, used to create the internal-directory accounts via the # management REST API / CLI. Secret is a SHA-512-crypt hash, injected via @@ -102,8 +195,10 @@ in { secret = "%{file:/run/credentials/stalwart.service/stalwart-admin}%"; }; - # Route docs routing: /strategy.md + /routing.md: - # local domains → local store, everything else → MX. local/mx are built-in. + # Route docs mta/outbound/routing.md + strategy.md: local domains → local + # store, everything else → MX. The strategy names are defined explicitly + # (docs define queue.route.mx/local; the built-in fallback in + # core.rs:get_route_or_default only kicks in for undeclared names). queue.strategy.route = [ { "if" = "is_local_domain('', rcpt_domain)"; @@ -111,6 +206,13 @@ in { } {"else" = "'mx'";} ]; + queue.route."mx" = { + type = "mx"; + ip-lookup = "ipv4_then_ipv6"; + }; + queue.route."local" = { + type = "local"; + }; }; }; @@ -118,8 +220,8 @@ in { # ProtectSystem=strict). The TLS key tlsa-updater installs is 0640 root:root; # regrant it to the stalwart group after every cert sync so stalwart can serve TLS. systemd.services.stalwart = { - after = ["tlsa-update.service" "stalwart-admin-secret.service"]; - requires = ["tlsa-update.service" "stalwart-admin-secret.service"]; + after = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"]; + requires = ["tlsa-update.service" "stalwart-admin-secret.service" "stalwart-dkim.service"]; }; # Make the management CLI available for account creation and maildir import @@ -142,6 +244,17 @@ in { }; }; + # Install the outbound DKIM key into the stalwart data dir before the service starts. + systemd.services.stalwart-dkim = { + description = "Install Stalwart outbound DKIM key"; + wantedBy = ["multi-user.target"]; + before = ["stalwart.service"]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${writeDkimKey}"; + }; + }; + # LoadCredential: expose the materialized hash to stalwart only at # /run/credentials/stalwart.service/stalwart-admin (see `credentials` option # in the upstream module; the value is the source path on disk). @@ -172,7 +285,11 @@ in { "/var/lib/bulwark:/app/data:Z" ]; environment = { - JMAP_SERVER_URL = "http://127.0.0.1:8080"; + # JMAP_SERVER_URL is the public origin (the browser uses it verbatim for + # /.well-known/jmap + the session apiUrl). HOSTNAME stays 127.0.0.1 so + # Next.js binds to loopback (caddy reverse_proxy's 127.0.0.1:3002); binding + # to the public hostname made the container unreachable for caddy (502). + JMAP_SERVER_URL = "https://mail.severijnse.eu"; HOSTNAME = "127.0.0.1"; PORT = "3002"; }; diff --git a/servers/hetzner/modules/services/tlsa-updater.nix b/servers/hetzner/modules/services/tlsa-updater.nix index 8693abf..a21d770 100644 --- a/servers/hetzner/modules/services/tlsa-updater.nix +++ b/servers/hetzner/modules/services/tlsa-updater.nix @@ -1,11 +1,15 @@ {pkgs, ...}: let # Caddy's canonical certificate storage (XDG data dir). Renewals land here, - # owned caddy:caddy 0600 — the mail server's non-root Postfix/Dovecot cannot - # read it directly, so we copy it into a world-readable distribution dir. + # owned caddy:caddy 0600 — stalwart cannot read it directly, so we copy it + # into a distribution dir stalwart's service (user "stalwart") can reach. caddyCertDir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu"; - # World-readable distribution dir mounted (RO) into the mail server container. + # Distribution dir read by stalwart (cert 0644, private key regranted to the + # "stalwart" group by systemd.services.stalwart-cert-perm). distCertDir = "/var/lib/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mail.severijnse.eu"; zoneFile = "/var/lib/coredns/zones/severijnse.eu.db"; + # Records the SPKI hash applied at the last restart of stalwart, so cert + # renewals trigger exactly one restart and unchanged certs never do. + stateFile = "/var/lib/tlsa-update/.last-spki"; syncScript = pkgs.writeShellScript "tlsa-update" '' set -euo pipefail @@ -22,11 +26,11 @@ exit 0 fi - # 1) Propagate Caddy's renewed certificate into the distribution dir the - # mail server mounts. Caddy stores certs 0600 caddy:caddy. The cert is - # world-readable (Postfix/Dovecot read it as root before dropping - # privileges); the private key is restricted to root (0640) so it is not - # exposed to other local users. + # 1) Propagate Caddy's renewed certificate into the distribution dir stalwart + # reads. Caddy stores certs 0600 caddy:caddy. The cert is world-readable; + # the private key is restricted to root (0640) and stalwart-cert-perm + # regrants it to the "stalwart" group so it is not exposed to other local + # users. install -D -m 0644 "$SRC_CERT" "$DST_CERT" install -D -m 0640 "$SRC_KEY" "$DST_KEY" @@ -47,8 +51,24 @@ # 4) Reload services so the changes take effect immediately. systemctl reload coredns.service || true - podman exec mailserver postfix reload || true - podman exec mailserver dovecot reload || true + # Stalwart reads its TLS certs ($certDir) via %{file:...}% placeholders at + # startup only; there is no signal-based reload (management API reload needs + # admin credentials we must not store). Restart it, but only when the cert + # actually changed (SPKI hash differs from the last applied run), so the + # daily timer and inotify events for unchanged certs do not drop connections. + # NOTE: stalwart.service `requires` this unit, so the restart must be issued + # with --no-block (async): a synchronous restart waits for stalwart to come + # back up, which in turn waits for this unit to finish — a deadlock. The state + # file is updated BEFORE the restart so the tlsa-update run that stalwart's + # required-activation re-triggers sees a matching hash and exits immediately. + if [ "$(cat "${stateFile}" 2>/dev/null || true)" != "$HEX" ]; then + printf '%s\n' "$HEX" > "${stateFile}" + # Restarting stalwart re-runs its Requires=tlsa-update dependency; guard + # with is-active so a boot-time run never races stalwart's initial start. + if systemctl is-active --quiet stalwart.service 2>/dev/null; then + systemctl --no-block restart stalwart.service || true + fi + fi echo "tlsa-update: TLSA set to $HEX" ''; @@ -57,6 +77,7 @@ in { # Ensure the distribution dir exists (Caddy does not write here). tmpfiles.rules = [ "d ${distCertDir} 0755 root root - -" + "d /var/lib/tlsa-update 0755 root root - -" ]; services.tlsa-update = { @@ -64,7 +85,7 @@ in { after = ["caddy.service" "coredns.service"]; partOf = ["coredns.service"]; wantedBy = ["multi-user.target"]; - path = with pkgs; [openssl coreutils gnused podman systemd]; + path = with pkgs; [openssl coreutils gnused systemd]; serviceConfig = { Type = "oneshot"; ExecStart = "${syncScript}";