.env.example

@@ -82,55 +82,3 @@ STALWART_SMARTHOST=
 # a certbot-issued cert (see config/config.toml [certificate]).
 STALWART_ACME_PROVIDER=
 STALWART_ACME_TOKEN=
-
-
-# ============================================================================
-# AUTHELIA — merged from /opt/authelia. Shares TS_OAUTH_CLIENT_SECRET, TS_TAILNET,
-# DB_MAGIC_NAME, REDIS_MAGIC_NAME from the stalwart section above.
-# ============================================================================
-# ============================================================================
-# authelia — operator configuration  (SSO / 2FA / OIDC, Tailscale sidecar)
-# ============================================================================
-# Copy to .env and fill in. Self-contained stack at /opt/authelia, outside any
-# upstream repo. Reuses the shared tailnet Postgres + Redis + SMTP relay.
-#
-# Authelia is a forward-auth / OIDC provider: it runs tailnet-only and the main
-# box Caddy fronts its portal (auth.<domain>) and routes protected vhosts
-# through its /api/verify endpoint. See caddy-forward-auth.snippet.
-# ============================================================================
-
-# Tailscale — same OAuth client as the rest of the tailnet (tag:authelia).
-AUTHELIA_MAGIC_NAME=authelia
-
-# Identity / cookie scope. Portal lives at AUTHELIA_PORTAL_URL; the session
-# cookie is valid across *.AUTHELIA_DOMAIN.
-AUTHELIA_DOMAIN=infinidim.net
-AUTHELIA_PORTAL_URL=https://auth.infinidim.net
-
-# Postgres (shared) — storage backend (users' 2FA devices, identity, consent).
-AUTHELIA_DB_NAME=authelia
-AUTHELIA_DB_USER=authelia
-# Generate: openssl rand -hex 24
-AUTHELIA_DB_PASSWORD=
-
-# Redis (shared) — session backend. Dedicated logical DB index.
-AUTHELIA_REDIS_DB=4
-
-# SMTP relay (shared) — password-reset + 2FA notifications.
-SMTP_HOST=smtp.example.com
-SMTP_PORT=587
-SMTP_USER=
-SMTP_PASSWORD=
-AUTHELIA_SMTP_SENDER=no-reply@infinidim.net
-
-# Authelia secrets. Generate each: openssl rand -hex 32
-AUTHELIA_SESSION_SECRET=
-AUTHELIA_STORAGE_ENCRYPTION_KEY=
-AUTHELIA_JWT_SECRET=
-AUTHELIA_OIDC_HMAC_SECRET=
-
-# First admin (web portal login). The hash goes in config/users_database.yml:
-#   docker run --rm authelia/authelia:latest \
-#     authelia crypto hash generate argon2 --password 'YOURPASS'
-AUTHELIA_ADMIN_USER=admin
-AUTHELIA_ADMIN_PASSWORD=

.gitignore

@@ -23,11 +23,6 @@ export/
 # config and is secret-free (DB password comes from $STALWART_DB_PASSWORD via
 # the EnvironmentVariable secret type). Don't add it here.
 
-# Authelia (moved under authelia/) — single .env lives at repo root; authelia/.env
-# is a symlink to it (matched by the .env rule above). Plus its runtime data.
-authelia/config/*.db
-authelia/config/notification.txt
-
 # Editor swap / backup files
 *.swp
 *.swo

LESSONS.md

@@ -308,50 +308,3 @@ signature, not republish anything. Two parts:
 `mail.tail7b1641.ts.net` (auth `stalwart-relay@waynehayes.com`), which re-emits
 as `mail.waynehayes.com` (`216.189.156.74` / `2602:ffc5:20::1:6b52`). That relay
 IP is why SPF needed `include:waynehayes.com` (#14 / the SPF fix).
-
-## 16. After a reboot, Stalwart started before Garage — admin site 404'd (NOT a boot loop)
-
-**Symptom:** Post-reboot, the Stalwart web admin / app assets wouldn't load (404 /
-blank), even though the container was `running` and **not** restart-looping.
-
-**Cause:** the web UI (and other app assets) live in the **S3 blob store (Garage)** —
-Stalwart unpacks/serves them from S3. On reboot Stalwart came up *before* Garage was
-ready, so the asset fetch failed. Stalwart itself was fine (PG connected, listeners up);
-only the S3-backed content was missing. Easy to misread as "Stalwart is broken."
-
-**Fix:** once Garage is up, restart Stalwart (or it picks them up on the next fetch).
-Quick confirm it's a backend-readiness issue, not Stalwart: `running`+`healthy` but assets
-404 → probe the backend from the sidecar (`nc -z garage.<tailnet> 3900`).
-
-**Rule for the whole fleet (federatedSocial):** every app must gate on its backends being
-**live, not merely present**. Model it on the Stalwart sidecar's healthcheck —
-`depends_on: { <backend>: service_healthy }` plus a check that actually *probes* PG/Redis/
-Garage over the tailnet (see #1, the PG-startup-race healthcheck). Don't assume shared
-infra boots first; make it a startup-ordering/readiness convention across all sidecars.
-
-## 17. A flapping shared-store consumer (atuin) looked like a Postgres problem
-
-**Symptom:** "Postgres seems to be the cause / unstable." Actually `atuin-server` had
-**RestartCount 6318, exit 1** — crash-looping for days and generating all the noise.
-
-**Cause:** atuin couldn't reach/authenticate its DB and crash-looped under
-`restart: unless-stopped`. **Postgres itself was healthy** (6 days up, 0 restarts,
-17/100 conns). atuin never even established a connection — *no* atuin lines in the PG log
-and *no* atuin rows in `pg_stat_activity` — i.e. it was dying **before** reaching PG.
-
-**Diagnosis (fast):**
-```bash
-# which container is actually flapping (PG health != consumer health):
-docker inspect <c> --format '{{.RestartCount}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}'
-# is a consumer reconnect-storming the shared store? distinct/ghost client_addr = churn:
-docker exec <pg> psql -U postgres -tAc \
-  "SELECT client_addr, state, count(*) FROM pg_stat_activity GROUP BY 1,2 ORDER BY 1"
-```
-Ephemeral sidecar nodes get a **new tailnet IP per restart**, so successive incarnations
-leave **ghost idle connections** from dead IPs — a handy "how many times did it restart"
-fingerprint (we saw this with Stalwart too: 1 live IP + 2 ghosts).
-
-**Rule for the whole fleet:** a shared Postgres/Redis/Garage is a blast-radius surface —
-one misconfigured consumer shouldn't be mistaken for a shared-infra outage. Confirm a
-consumer's creds + backend reachability **before** enabling `restart: always/unless-stopped`,
-and when something "looks like the DB," check the *consumers* first.

authelia/CLAUDE.md

@@ -1,83 +0,0 @@
-# CLAUDE.md — authelia
-
-Guidance for Claude Code in this repo. Read before editing.
-
-## What this is
-
-A standalone [Authelia](https://www.authelia.com/) deployment — SSO portal, 2FA,
-and (optional) OIDC provider — as a Tailscale sidecar, plugged into the shared
-tailnet **Postgres** (storage) and **Redis** (sessions), with the shared **SMTP
-relay** for reset/2FA mail. Sibling to `tailwart`; same pattern, different job.
-Domain: `infinidim.net` (portal `auth.infinidim.net`).
-
-Self-contained and **outside** any upstream repo (own `.env`, compose, config,
-ACL snippet). Reads only the tailnet at runtime.
-
-## Architecture
-
-```
-  main box Caddy (tag:reverse-proxy)            tailnet-only auth
-  ┌────────────────────────────┐               ┌───────────────────────┐
-  │ auth.infinidim.net  ───────┼──────────────▶│ ts-authelia sidecar    │
-  │   reverse_proxy :9091      │   tailnet     │  authelia (no WAN, no  │
-  │ protected vhosts:          │               │  host ports) :9091     │
-  │   forward_auth → :9091     │               └──────────┬─────────────┘
-  └────────────────────────────┘                          │
-                                                   ┌───────┴───────┐
-                                                   ▼       ▼       ▼
-                                              Postgres  Redis   SMTP relay
-```
-
-- **Sidecar** (`tag:authelia`), `network_mode: service:ts-authelia`, no host ports.
-- Authelia listens `:9091` on the tailnet. The **main box Caddy** (layer 7, not
-  a separate edge) fronts the portal and wires `forward_auth` for protected
-  services — see `caddy-forward-auth.snippet`. This is ordinary L7, unlike
-  tailwart's L4 mail edge.
-- Storage → Postgres (`authelia` role/db), sessions → Redis (logical DB
-  `AUTHELIA_REDIS_DB`), mail → shared relay.
-
-## The `.env` contract
-
-`.env` (gitignored) is the whole surface. Secrets and infra hostnames reach
-Authelia as `AUTHELIA_*` **env overrides** (set in `docker-compose.yml`), which
-take precedence over `config/configuration.yml`. That keeps the committed yml
-free of secrets and MagicDNS names. The yml holds only non-secret structure
-(access_control, session.cookies, totp, regulation).
-
-## Prerequisites (shared tailnet infra)
-
-1. Postgres role+db: `authelia` / `AUTHELIA_DB_NAME`. Create via a one-off
-   `CREATE ROLE authelia LOGIN PASSWORD '…'; CREATE DATABASE authelia OWNER
-   authelia;` against `the-record-prod` (the federatedSocial shared Postgres).
-2. Redis: nothing to create — uses logical DB index `AUTHELIA_REDIS_DB` (4) so
-   it won't collide with the fediverse apps or tailwart (which uses 3).
-3. `config/users_database.yml`: at least one user with an argon2id hash:
-   `docker run --rm authelia/authelia:latest authelia crypto hash generate
-   argon2 --password 'PASS'`.
-4. Admin console: assign `tag:authelia` to the OAuth client (Devices/Core +
-   Keys/AuthKeys) and add `acl-snippet.hujson` to the policy.
-5. Main box Caddy: add `caddy-forward-auth.snippet` (portal vhost + the
-   `(authelia)` import) and a cert for `auth.infinidim.net`.
-
-## Pitfalls
-
-- **Authelia config schema drifts hard between minor versions** (4.37→4.38 moved
-  session to `cookies[]`, added `identity_validation`, changed `storage.postgres`
-  to `address`). The image is **pinned to 4.39.20** and the yml passes
-  `authelia config validate` against it. Re-validate before bumping the tag:
-  `docker run --rm -e ... -v ./config:/config:ro authelia/authelia:<tag> authelia config validate`.
-- **Don't put secrets in `configuration.yml`.** Use the `AUTHELIA_*` env path.
-- **Redis DB index collisions.** tailwart=3, authelia=4. Keep them distinct.
-- **Postgres password drift.** Role passwords only apply on `CREATE`/`ALTER`;
-  test auth over the **tailnet** (scram), never `127.0.0.1` (that pg_hba line is
-  `trust` and accepts any password — it'll lie to you).
-- **forward_auth is layer 7**, on the main Caddy. Don't confuse it with
-  tailwart's L4 edge — different mechanism entirely.
-
-## What not to do
-
-- Don't write into `/opt/federatedSocial` (read its `.env` if needed).
-- Don't add `ports:` to the Authelia container — the main Caddy is the only
-  public path in.
-- Don't commit `.env`.
-- Don't break the sidecar netns boundary.

authelia/README.md

@@ -1,52 +0,0 @@
-# authelia
-
-> SSO / 2FA / OIDC for `infinidim.net`, as a tailnet sidecar. Storage in
-> Postgres, sessions in Redis, mail via the shared relay — no WAN presence;
-> the main box Caddy fronts the portal and gates protected vhosts.
-
-Standalone sibling to [tailwart](/opt/tailwart). See [CLAUDE.md](./CLAUDE.md).
-
-## Layout
-
-```
-authelia/
-├── docker-compose.yml          # ts-authelia sidecar + authelia
-├── config/
-│   ├── configuration.yml       # non-secret structure (4.38 strawman)
-│   └── users_database.yml      # file backend — admin user (argon2id)
-├── caddy-forward-auth.snippet  # portal vhost + (authelia) import for box Caddy
-├── acl-snippet.hujson          # tag:authelia owner + backend/edge grants
-├── .env.example                # operator surface
-└── .gitignore
-```
-
-## Quickstart
-
-```bash
-cp .env.example .env && $EDITOR .env       # (a generated .env is already here)
-
-# 1. create the authelia role/db in shared Postgres:
-docker exec -i federated-shared-db-postgres-1 psql -U postgres <<'SQL'
-DO $$ BEGIN
-  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='authelia') THEN
-    CREATE ROLE authelia LOGIN PASSWORD 'PASTE_AUTHELIA_DB_PASSWORD';
-  END IF;
-END $$;
-SELECT 'CREATE DATABASE authelia OWNER authelia'
- WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname='authelia')\gexec
-SQL
-
-# 2. admin console: assign tag:authelia to the OAuth client + paste acl-snippet
-# 3. bring up (tailnet-only)
-docker compose up -d
-# 4. add caddy-forward-auth.snippet to the main box Caddy + a cert for auth.infinidim.net
-```
-
-Then log in at `https://auth.infinidim.net` with the admin user from `.env`.
-
-## Status
-
-Pinned to Authelia **4.39.20**. `configuration.yml` passes `authelia config
-validate` against that image with the real env — schema and secrets check out.
-Runtime backends (Postgres/Redis/SMTP connectivity) get exercised on the first
-`docker compose up`.

authelia/acl-snippet.hujson

@@ -1,18 +0,0 @@
-// authelia — merge into your live Tailscale policy (admin console). Snippet only.
-
-// 1) tagOwners — self-ownership required for auth-key node creation:
-//    "tag:authelia": ["autogroup:admin", "tag:authelia"],
-
-// 2) grants — Authelia reaches its two shared backends:
-{ "src": ["tag:authelia"], "dst": ["tag:db-postgres"], "ip": ["tcp:5432"] },
-{ "src": ["tag:authelia"], "dst": ["tag:db-redis"],    "ip": ["tcp:6379"] },
-
-// 3) grant — the main Caddy (tag:reverse-proxy) reaches the portal/verify API.
-{ "src": ["tag:reverse-proxy"], "dst": ["tag:authelia"], "ip": ["tcp:9091"] },
-
-// 4) admin console (not this file): assign tag:authelia to the same OAuth
-//    client, on the Devices/Core + Keys/AuthKeys scopes.
-
-// NOTE on SMTP: the sidecar reaches the relay (smtp.infinidim.net:587) the same
-// way the fediverse apps do. If that relay is a TAILNET host (not public),
-// add: { "src": ["tag:authelia"], "dst": ["<relay-tag-or-host>"], "ip": ["tcp:587"] }

authelia/caddy-forward-auth.snippet

@@ -1,35 +0,0 @@
-# Wired into the MAIN box Caddyfile (/etc/caddy/Caddyfile). Authelia is layer 7,
-# so unlike tailwart's mail edge it's an ordinary reverse_proxy + forward_auth.
-# This file mirrors what is deployed. Upstream host = AUTHELIA_MAGIC_NAME, the
-# Authelia node's MagicDNS name (currently `agrajag`).
-#
-# Two deliberate differences from a vanilla Authelia example, to match this host:
-#  - No explicit `tls` cert files: this Caddy uses automatic HTTPS like its other
-#    vhosts (there is no /etc/caddy/certs). ACME for auth.infinidim.net works via
-#    the :443→:8443 SNI fan-out (tls-alpn-01) and :80 (http-01).
-#    NOTE: infinidim.net has a CAA record pinning issuance to Let's Encrypt by
-#    accounturi — this Caddy's LE account must be listed there or issuance 403s
-#    ("CAA record prevents issuance"). Stalwart's account + this Caddy's account
-#    are both allowlisted.
-#  - Endpoint is /api/authz/forward-auth (Authelia 4.38+/4.39 path). The legacy
-#    /api/verify?rd=... is deprecated; the portal redirect comes from
-#    `authelia_url` in configuration.yml, so no ?rd= query is needed.
-
-# 1) The Authelia portal itself (access_control marks it `bypass`).
-auth.infinidim.net {
-    reverse_proxy agrajag.tail7b1641.ts.net:9091
-}
-
-# 2) A reusable forward-auth snippet — import it into any vhost you want SSO on.
-(authelia) {
-    forward_auth agrajag.tail7b1641.ts.net:9091 {
-        uri /api/authz/forward-auth
-        copy_headers Remote-User Remote-Groups Remote-Name Remote-Email
-    }
-}
-
-# 3) Example protected service: gate it behind Authelia, then proxy the backend.
-# secure-app.infinidim.net {
-#     import authelia
-#     reverse_proxy some-backend.tail7b1641.ts.net:8080
-# }

authelia/config/configuration.yml

@@ -1,73 +0,0 @@
-# ============================================================================
-# Authelia config — non-secret structure only. Secrets + infra hosts (Postgres
-# address, Redis host, SMTP) are injected via AUTHELIA_* env from the compose
-# file, so this stays commit-safe.
-# ============================================================================
-# STRAWMAN. Authelia's schema moves between releases (this targets 4.38+: note
-# session.cookies[], identity_validation, storage.postgres.address). Verify
-# against the pinned image tag before trusting it, then pin the tag.
-
-theme: dark
-
-server:
-  address: tcp://0.0.0.0:9091
-
-log:
-  level: info
-
-totp:
-  issuer: infinidim.net
-
-# File-based user database (simple, good for play). Swap to LDAP later if needed.
-authentication_backend:
-  file:
-    path: /config/users_database.yml
-    password:
-      algorithm: argon2
-
-access_control:
-  default_policy: deny
-  rules:
-    # The portal itself must always be reachable.
-    - domain: auth.infinidim.net
-      policy: bypass
-    # Everything else under the domain needs at least one factor.
-    - domain: "*.infinidim.net"
-      policy: one_factor
-
-session:
-  # secret + redis host/port/db come from env (AUTHELIA_SESSION_*).
-  cookies:
-    - domain: infinidim.net
-      authelia_url: https://auth.infinidim.net
-      default_redirection_url: https://infinidim.net
-
-regulation:
-  max_retries: 3
-  find_time: 2m
-  ban_time: 5m
-
-# storage (postgres), notifier (smtp), session.secret/redis, and
-# identity_validation.reset_password.jwt_secret all arrive via env — see
-# docker-compose.yml. Kept out of this file to avoid duplicating infra
-# hostnames and to keep secrets off disk in the repo.
-
-# ----------------------------------------------------------------------------
-# OIDC provider (optional) — uncomment to let apps log in via OpenID Connect.
-# Needs AUTHELIA_OIDC_HMAC_SECRET and a JWKS signing key:
-#   docker run --rm authelia/authelia:latest \
-#     authelia crypto certificate rsa generate --directory /tmp && cat /tmp/...
-# ----------------------------------------------------------------------------
-# identity_providers:
-#   oidc:
-#     hmac_secret: <env AUTHELIA_OIDC_HMAC_SECRET>
-#     jwks:
-#       - key: |
-#           -----BEGIN PRIVATE KEY----- ...
-#     clients:
-#       - client_id: stalwart
-#         client_name: Stalwart Mail
-#         client_secret: '<hashed>'
-#         redirect_uris:
-#           - https://mail.infinidim.net/...
-#         scopes: [openid, profile, email]

authelia/config/users_database.yml

@@ -1,11 +0,0 @@
-# Authelia file user backend. Regenerate a hash with:
-#   docker run --rm authelia/authelia:4.39.20 authelia crypto hash generate argon2 --password 'PASS'
-# Username/password come from AUTHELIA_ADMIN_USER / AUTHELIA_ADMIN_PASSWORD in the root .env.
-users:
-  zarniwoop:
-    disabled: false
-    displayname: "Zarniwoop"
-    password: "$argon2id$v=19$m=65536,t=3,p=4$2Zh5wh3yN/kCvR26mTRo9Q$g86OC3E8Q4lH0czTOal7Gci2+U6t0ZIFhogIwtRoA5M"
-    email: admin@infinidim.net
-    groups:
-      - admins

authelia/docker-compose.yml

@@ -1,71 +0,0 @@
-# authelia — SSO / 2FA / OIDC provider as a Tailscale sidecar (NO WAN presence).
-#
-# Storage → shared Postgres, sessions → shared Redis, mail → shared SMTP relay,
-# all over the tailnet via MagicDNS. The portal (auth.infinidim.net) and any
-# forward-auth-protected vhosts are wired on the main box Caddy — see
-# caddy-forward-auth.snippet.
-#
-# Prereq: the `authelia` Postgres role/db (see README). Bring up: docker compose up -d
-
-name: authelia
-
-services:
-
-  ts-authelia:
-    image: tailscale/tailscale:latest
-    hostname: ${AUTHELIA_MAGIC_NAME}
-    environment:
-      TS_AUTHKEY: ${TS_OAUTH_CLIENT_SECRET}?ephemeral=true
-      TS_EXTRA_ARGS: --advertise-tags=tag:authelia
-      TS_HOSTNAME: ${AUTHELIA_MAGIC_NAME}
-      TS_ACCEPT_DNS: "true"
-      TS_AUTH_ONCE: "true"
-      TS_USERSPACE: "false"
-      TS_ENABLE_HEALTH_CHECK: "true"
-      TS_LOCAL_ADDR_PORT: "127.0.0.1:9002"
-    dns: [1.1.1.1, 1.0.0.1]
-    devices:
-      - /dev/net/tun:/dev/net/tun
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-    healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9002/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 6
-      start_period: 30s
-    restart: unless-stopped
-
-  authelia:
-    image: authelia/authelia:4.39.20
-    network_mode: "service:ts-authelia"
-    environment:
-      X_AUTHELIA_CONFIG: /config/configuration.yml
-      # Secrets + infra hosts via env so configuration.yml stays commit-safe and
-      # free of hardcoded MagicDNS names. Env overrides win over the yml.
-      AUTHELIA_SESSION_SECRET: ${AUTHELIA_SESSION_SECRET}
-      AUTHELIA_STORAGE_ENCRYPTION_KEY: ${AUTHELIA_STORAGE_ENCRYPTION_KEY}
-      AUTHELIA_IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET: ${AUTHELIA_JWT_SECRET}
-      AUTHELIA_STORAGE_POSTGRES_ADDRESS: tcp://${DB_MAGIC_NAME}.${TS_TAILNET}:5432
-      AUTHELIA_STORAGE_POSTGRES_DATABASE: ${AUTHELIA_DB_NAME}
-      AUTHELIA_STORAGE_POSTGRES_USERNAME: ${AUTHELIA_DB_USER}
-      AUTHELIA_STORAGE_POSTGRES_PASSWORD: ${AUTHELIA_DB_PASSWORD}
-      AUTHELIA_SESSION_REDIS_HOST: ${REDIS_MAGIC_NAME}.${TS_TAILNET}
-      AUTHELIA_SESSION_REDIS_PORT: "6379"
-      AUTHELIA_SESSION_REDIS_DATABASE_INDEX: ${AUTHELIA_REDIS_DB}
-      AUTHELIA_NOTIFIER_SMTP_ADDRESS: submission://${SMTP_HOST}:${SMTP_PORT}
-      AUTHELIA_NOTIFIER_SMTP_USERNAME: ${SMTP_USER}
-      AUTHELIA_NOTIFIER_SMTP_PASSWORD: ${SMTP_PASSWORD}
-      AUTHELIA_NOTIFIER_SMTP_SENDER: ${AUTHELIA_SMTP_SENDER}
-    volumes:
-      - ./config/configuration.yml:/config/configuration.yml:ro
-      - ./config/users_database.yml:/config/users_database.yml:ro
-      - authelia-data:/data
-    depends_on:
-      ts-authelia:
-        condition: service_healthy
-    restart: unless-stopped
-
-volumes:
-  authelia-data: