diff --git a/nixos/roles/matrix/postgres-backup.nix b/nixos/roles/matrix/postgres-backup.nix new file mode 100644 index 0000000..752acd2 --- /dev/null +++ b/nixos/roles/matrix/postgres-backup.nix @@ -0,0 +1,102 @@ +{ + config, + pkgs, + lib, + ... +}: +{ + sops.secrets = { + pg_replication_password = { + owner = "postgres"; + group = "postgres"; + }; + }; + + services.postgresql = { + enable = true; + package = pkgs.postgresql_15; # must match cyper-proxy's PG version exactly + + settings = { + hot_standby = true; + hot_standby_feedback = true; + max_standby_streaming_delay = "30s"; + listen_addresses = "127.0.0.1"; # only local, no external exposure + wal_receiver_timeout = "60s"; + recovery_min_apply_delay = "0"; # set e.g. "1h" for a delayed safety replica + }; + }; + + # Writes standby.signal and primary_conninfo before PostgreSQL starts. + systemd.services.postgresql = { + preStart = lib.mkAfter '' + DATADIR="${config.services.postgresql.dataDir}" + PG_PASS=$(cat ${config.sops.secrets.pg_replication_password.path}) + + # Tell Postgres to start as a hot standby + if [ ! -f "$DATADIR/standby.signal" ]; then + touch "$DATADIR/standby.signal" + chown postgres:postgres "$DATADIR/standby.signal" + fi + + # primary_conninfo via Tailscale — no public IP involved + CONNINFO="host=100.109.10.91 port=5432 user=replicator password=$PG_PASS application_name=cyper-controller sslmode=require" + + grep -v "^primary_conninfo" "$DATADIR/postgresql.auto.conf" 2>/dev/null > /tmp/auto.conf.tmp || true + echo "primary_conninfo = '$CONNINFO'" >> /tmp/auto.conf.tmp + mv /tmp/auto.conf.tmp "$DATADIR/postgresql.auto.conf" + chown postgres:postgres "$DATADIR/postgresql.auto.conf" + ''; + }; + + # Run once manually to seed the standby: systemctl start postgresql-basebackup + # Do NOT add it to wantedBy — it would wipe the data dir on every reboot. + systemd.services.postgresql-basebackup = { + description = "Bootstrap PostgreSQL standby via pg_basebackup from cyper-proxy"; + requires = [ + "network-online.target" + "tailscaled.service" + ]; + after = [ + "network-online.target" + "tailscaled.service" + ]; + + serviceConfig = { + Type = "oneshot"; + User = "postgres"; + RemainAfterExit = false; + }; + + script = '' + DATADIR="${config.services.postgresql.dataDir}" + PG_PASS=$(cat ${config.sops.secrets.pg_replication_password.path}) + + if [ -d "$DATADIR/global" ]; then + echo "Data directory already exists — skipping." + echo "Remove $DATADIR manually to force a re-initialise." + exit 0 + fi + + echo "Running pg_basebackup from cyper-proxy via Tailscale..." + PGPASSWORD="$PG_PASS" ${config.services.postgresql.package}/bin/pg_basebackup \ + --host=100.109.10.91 \ + --port=5432 \ + --username=replicator \ + --pgdata="$DATADIR" \ + --wal-method=stream \ + --write-recovery-conf \ + --checkpoint=fast \ + --progress \ + --verbose + + chown -R postgres:postgres "$DATADIR" + chmod 0700 "$DATADIR" + echo "Done. Start postgresql.service to begin streaming replication." + ''; + }; + + # Block any external access to postgres on the public interface + networking.firewall = { + interfaces."tailscale0".allowedTCPPorts = [ ]; # replication is outbound only + }; +} diff --git a/nixos/roles/matrix/synapse.nix b/nixos/roles/matrix/synapse.nix index ca57fa7..47318fd 100644 --- a/nixos/roles/matrix/synapse.nix +++ b/nixos/roles/matrix/synapse.nix @@ -34,6 +34,7 @@ in owner = "matrix-synapse"; group = "matrix-synapse"; }; + pg_replication_password = { }; }; services = { @@ -145,17 +146,34 @@ in enable = true; initialScript = pkgs.writeText "synapse-init.sql" '' CREATE ROLE "matrix-synapse" WITH LOGIN PASSWORD 'synapse'; + CREATE ROLE replicator WITH REPLICATION LOGIN; CREATE DATABASE "matrix-synapse" WITH OWNER "matrix-synapse" TEMPLATE template0 LC_COLLATE = "C" LC_CTYPE = "C"; ''; + + settings = { + wal_level = "replica"; + max_wal_senders = 3; + wal_keep_size = "512MB"; + }; + + authentication = lib.mkAfter '' + host replication replicator 100.0.0.0/8 scram-sha-256 + ''; }; }; - systemd.services.matrix-synapse.serviceConfig.ReadOnlyPaths = [ - "/var/lib/mautrix-discord" - "/var/lib/mautrix-whatsapp" - ]; - + systemd.services = { + matrix-synapse.serviceConfig.ReadOnlyPaths = [ + "/var/lib/mautrix-discord" + "/var/lib/mautrix-whatsapp" + ]; + postgresql.postStart = lib.mkAfter '' + PG_PASS=$(cat ${config.sops.secrets.pg_replication_password.path}) + ${config.services.postgresql.package}/bin/psql -U postgres -c \ + "ALTER ROLE replicator WITH PASSWORD '$PG_PASS';" + ''; + }; } diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml index ca56938..895789d 100644 --- a/secrets/secrets.yaml +++ b/secrets/secrets.yaml @@ -13,6 +13,7 @@ mjolnir_access_token: ENC[AES256_GCM,data:vvrAY9CAkEIGEzah+TQiwa6PahGuXVvU7wzBpT coturn_static_auth_secret: ENC[AES256_GCM,data:7AI0E8Hu4WxI5q4j1GqBMSQ+evE006uPMtwIfGn4eFz+XB2JA6fhhiGMPPxSkqOyK+3eZJ5ahiG05JpmBmmAbw==,iv:hQJQQDVo43U7lvV754PC1THeFCpZZEyag+BslXyoDos=,tag:Vkm+IXr1h8ZNpah6UYaKng==,type:str] discord_bot_token: ENC[AES256_GCM,data:j37Qo3FCyRwNFqWSWpnQKCs+AxH5HlQ8U5If7ylHilQoORp8Pb3TtNETTJSjZyvUXllldevAbHrbAEEKnNfoUJx1U8/wl6H0,iv:WQqxFXTE+0LIB2lSvVcnr4LNXPE7uzNc0Kk8NU6Z/aE=,tag:fNeQLhoThEgfa4sSGKLZCw==,type:str] discord_client_id: ENC[AES256_GCM,data:U/iUKXT6Nsl6LRN9lPh1xaIaqw==,iv:k7kQ8rJBrMs3YwD9aDfZ6qhd7H3aVsSPTOwEIxVTw2Y=,tag:2wKhxGbf+P+h3BYeWUSczA==,type:str] +pg_replication_password: ENC[AES256_GCM,data:w2h07D+j3LNkcbvoKQ2Qp3HSvC2Wf5HRAPAo/HNhmUkHBOaDyILNxo7IDjqajv0jytpG7q4joCJQhS7tEUlA9Q==,iv:26ZurAq61IDqGdAl0yPpoTJElo93hJJIEUlza4DGDNc=,tag:a46FOKgeqEEZE+rC+H9NbQ==,type:str] gitea: dbPassword: ENC[AES256_GCM,data:S6VvRgkdYk1AzXljyQEEq68UJ9zrFy6+INBMIAspXNcqcM6o+es19o0mcXA=,iv:/pHYpkZZq+9Md+75uSCb2YXfSvaDzUh6mMfH53wb7eg=,tag:ZnbyCQwrK2JnbO5HFqgJYw==,type:str] internalToken: ENC[AES256_GCM,data:7N8TkPNb1YdCk2uAcCvVd2pKRVOf85//DYxAvz0UCg1E8ccEI5630xVyKafDFiSTM4ER7xiYelartzXL0jLWSf3QNOjSHUP8TIAz4bJRAZUJPxO917bURSLGGe7WEOfONzqy3Ts5QhrJ,iv:DiIs1ytlwLvqD/Ejep6m2fmpSqdFZkxBcgLNt6+29jY=,tag:8jsEcOkH0p+1mP9cnVjiDQ==,type:str] @@ -31,7 +32,7 @@ sops: N3I5dzUwc3JtYzczMUhyT04vSHlZamMKT+FzYcDLmlEFYxm/XoBpJb8XaZzBH1v9 6fuez+zApathZfl14w41kAUojPWBznnxDqYtNvzVVLXwnpp3BMx+7w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-07T07:00:06Z" - mac: ENC[AES256_GCM,data:KSkcRm/aTGAZBfj2ZZ03x8EB2Sh0lFKUSDKLedgtYYk/QnUKTZOO8oaT36xIdrPN0pjK1CnElDQMkAHG6JCklif2UkcodKcerVWaVcNwZ4mk6wSvZz7OIqneMR0W/U+Ly3NMgwIKrlP9f7axiYMq9JyK6pVeepKrmw4RvOPzxqU=,iv:vlcFxxV5EofNAPnDf7eGJZ8FUM83uGUnkZtU57Epb3Y=,tag:yfYpa/F7PTwvZY11SZyRaw==,type:str] + lastmodified: "2026-05-09T09:12:42Z" + mac: ENC[AES256_GCM,data:sTVJcBb8cBzixOBQNlx44/m8W3smfwP5fhmnm9hlr5iwMuPJ7JeKTUqqlQaeL4RX/MpEuLc+Rm4thromJ11M/aA5yiqgWOY7vn8xYPoScGzx6HfV1cRJTofmrWmpxrDICQULwOaO+c8vwFBPy7fVqF/AacRtejx5sEOxsMzrYR8=,iv:/Fc5//8coI/rdQIyGcxCTgXPzOS9xNd0ChDHNs4yffw=,tag:8w6bbZcWMBZQWkujhXQY0w==,type:str] unencrypted_suffix: _unencrypted version: 3.12.2