diff --git a/hosts/cyper-controller/configuration.nix b/hosts/cyper-controller/configuration.nix index ee2be17..750ed91 100644 --- a/hosts/cyper-controller/configuration.nix +++ b/hosts/cyper-controller/configuration.nix @@ -13,6 +13,7 @@ ../../nixos/roles/frontpage # ../../nixos/roles/paperless-ngx.nix ../../nixos/roles/octoprint.nix + ../../nixos/roles/matrix/postgres-backup.nix ]; networking = { diff --git a/nixos/roles/gitea.nix b/nixos/roles/gitea.nix index c83fb2e..4d3a6b9 100644 --- a/nixos/roles/gitea.nix +++ b/nixos/roles/gitea.nix @@ -81,7 +81,7 @@ in lfs = { enable = true; - #contentDir = "${config.services.gitea.stateDir}/data/lfs"; + contentDir = "${config.services.gitea.stateDir}/data/lfs"; }; database = { @@ -104,11 +104,6 @@ in DISABLE_SSH = false; START_SSH_SERVER = true; LFS_START_SERVER = true; - LFS_JWT_SECRET_URI = "file://${config.sops.secrets."gitea/lfsJwtSecret".path}"; - }; - - lfs = { - PATH = "${config.services.gitea.stateDir}/data/lfs"; }; metrics = { diff --git a/nixos/roles/matrix/postgres-backup.nix b/nixos/roles/matrix/postgres-backup.nix index 752acd2..1bf7055 100644 --- a/nixos/roles/matrix/postgres-backup.nix +++ b/nixos/roles/matrix/postgres-backup.nix @@ -7,96 +7,108 @@ { sops.secrets = { pg_replication_password = { - owner = "postgres"; - group = "postgres"; + owner = "root"; + group = "root"; }; }; - services.postgresql = { - enable = true; - package = pkgs.postgresql_15; # must match cyper-proxy's PG version exactly + virtualisation.docker.enable = true; - settings = { - hot_standby = true; - hot_standby_feedback = true; - max_standby_streaming_delay = "30s"; - listen_addresses = "127.0.0.1"; # only local, no external exposure - wal_receiver_timeout = "60s"; - recovery_min_apply_delay = "0"; # set e.g. "1h" for a delayed safety replica - }; - }; - - # Writes standby.signal and primary_conninfo before PostgreSQL starts. - systemd.services.postgresql = { - preStart = lib.mkAfter '' - DATADIR="${config.services.postgresql.dataDir}" - PG_PASS=$(cat ${config.sops.secrets.pg_replication_password.path}) - - # Tell Postgres to start as a hot standby - if [ ! -f "$DATADIR/standby.signal" ]; then - touch "$DATADIR/standby.signal" - chown postgres:postgres "$DATADIR/standby.signal" - fi - - # primary_conninfo via Tailscale — no public IP involved - CONNINFO="host=100.109.10.91 port=5432 user=replicator password=$PG_PASS application_name=cyper-controller sslmode=require" - - grep -v "^primary_conninfo" "$DATADIR/postgresql.auto.conf" 2>/dev/null > /tmp/auto.conf.tmp || true - echo "primary_conninfo = '$CONNINFO'" >> /tmp/auto.conf.tmp - mv /tmp/auto.conf.tmp "$DATADIR/postgresql.auto.conf" - chown postgres:postgres "$DATADIR/postgresql.auto.conf" - ''; - }; - - # Run once manually to seed the standby: systemctl start postgresql-basebackup - # Do NOT add it to wantedBy — it would wipe the data dir on every reboot. - systemd.services.postgresql-basebackup = { - description = "Bootstrap PostgreSQL standby via pg_basebackup from cyper-proxy"; + systemd.services.postgresql-replica = { + description = "PostgreSQL WAL streaming replica (Docker)"; requires = [ - "network-online.target" + "docker.service" "tailscaled.service" + "network-online.target" ]; after = [ - "network-online.target" + "docker.service" "tailscaled.service" + "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; serviceConfig = { - Type = "oneshot"; - User = "postgres"; - RemainAfterExit = false; + Type = "simple"; + Restart = "on-failure"; + RestartSec = "10s"; }; + preStart = '' + DATADIR="/storage/backup/postgresql-replica" + PG_PASS=$(cat ${config.sops.secrets.pg_replication_password.path}) + + ${pkgs.docker}/bin/docker pull postgres:17 + + if [ ! -f "$DATADIR/PG_VERSION" ]; then + echo "No data dir found — running pg_basebackup..." + rm -rf "$DATADIR" + + ${pkgs.docker}/bin/docker run --rm \ + -e PGPASSWORD="$PG_PASS" \ + -v "/storage/backup:/out" \ + --network host \ + postgres:17 \ + pg_basebackup \ + --host=100.109.10.91 \ + --port=5432 \ + --username=replicator \ + --pgdata=/out/postgresql-replica \ + --wal-method=stream \ + --checkpoint=fast \ + --progress \ + --verbose + + # standby signal + touch "$DATADIR/standby.signal" + + # primary conninfo + cat > "$DATADIR/postgresql.auto.conf" < "$DATADIR/postgresql.conf" < "$DATADIR/postgresql.conf" </dev/null || true ''; }; - # Block any external access to postgres on the public interface - networking.firewall = { - interfaces."tailscale0".allowedTCPPorts = [ ]; # replication is outbound only + services.prometheus.exporters.postgres = { + enable = true; + port = 9188; + runAsLocalSuperUser = false; + dataSourceName = "postgresql://postgres@localhost:5434/postgres?sslmode=disable"; }; } diff --git a/nixos/roles/monitoring.nix b/nixos/roles/monitoring.nix index 4c9c14c..16e4479 100644 --- a/nixos/roles/monitoring.nix +++ b/nixos/roles/monitoring.nix @@ -134,7 +134,7 @@ in metrics_path = "/_synapse/metrics"; static_configs = [ { - targets = [ "127.0.0.1:9009" ]; + targets = [ "100.109.10.91:9009" ]; labels = { instance = config.networking.hostName; job = "master"; @@ -143,6 +143,28 @@ in } ]; } + { + job_name = "postgresql-replica"; + static_configs = [ + { + targets = [ "localhost:9188" ]; + labels = { + instance = config.networking.hostName; + }; + } + ]; + } + { + job_name = "postgresql-proxy"; + static_configs = [ + { + targets = [ "100.109.10.91:9188" ]; + labels = { + instance = "cyper-proxy"; + }; + } + ]; + } ] ++ (lib.mapAttrsToList mkNodeJob extraNodes) ++ (mkWeatherScrapeConfigs weatherCities); diff --git a/nixos/roles/nginx.nix b/nixos/roles/nginx.nix index ece3997..6792839 100644 --- a/nixos/roles/nginx.nix +++ b/nixos/roles/nginx.nix @@ -40,7 +40,11 @@ in virtualHosts = { # controller services (proxied to upstream tailscale node) - "git.cyperpunk.de" = mkProxy 9000; + "git.cyperpunk.de" = (mkProxy 9000) // { + extraConfig = '' + client_max_body_size 500m; + ''; + }; "search.cyperpunk.de" = mkProxy 11080; "file.cyperpunk.de" = mkProxy 10000; "ngx.cyperpunk.de" = mkWsProxy 28101;