From ee99a2da9db0b4f79e7893ecc5654307ef45c877 Mon Sep 17 00:00:00 2001 From: Samantha Atkins Date: Mon, 30 Mar 2026 20:32:28 +0000 Subject: [PATCH] reworked services --- CLAUDE.md | 93 ++++++++++++++++++++++++-------- proxmox/services/authentik.yml | 10 +++- proxmox/services/ghost.yml | 15 ++++-- proxmox/services/mariadb.yml | 5 +- proxmox/services/n8n.yml | 5 +- proxmox/services/nats.yml | 15 ++++-- proxmox/services/vaultwarden.yml | 6 ++- 7 files changed, 116 insertions(+), 33 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c6ce03e..0da4a5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,31 +4,18 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Overview -Proxmox VE homelab cluster running Docker Swarm services. Three Proxmox hosts (pve, adder, game) connected via VXLAN overlay on vmbr1 (10.10.10.0/24). VMs run Docker CE and form a swarm with pve-postgres as manager. +Proxmox VE homelab cluster running Docker Swarm services. Three Proxmox hosts connected via VXLAN overlay on vmbr1 (10.10.10.0/24). VMs run Docker CE and form a swarm with pve-postgres as manager. -## Deployment +## Hardware -All services deploy as Docker Swarm stacks from the manager node (pve-postgres). The active service definitions are in `proxmox/services/`. The `services/` directory contains older pre-migration copies. +| Node | Machine | RAM | Storage | Role | +|------|---------|-----|---------|------| +| pve (Meerkat) | Intel TNTGL357 mini PC | 64GB | 4TB NVMe | Proxmox host, data tier | +| adder | System76 Adder WS | 32GB | 2TB NVMe + RTX 2070 | Proxmox host, GPU/crypto tier | +| game (CyberPower) | Old gaming PC | 16GB | 500GB NVMe + 2TB HDD | Proxmox host | +| Synology NAS 425+ | — | — | 48TB usable | NFS backend (VM backups, ISOs, Nextcloud data — NOT live DBs) | -```bash -# Copy yml to manager and deploy (resolve secrets from pass inline) -scp proxmox/services/.yml pve-postgres:~/ -ssh pve-postgres "VAR=$(pass homelab/VAR) docker stack deploy -c .yml " - -# Remove a stack -ssh pve-postgres "docker stack rm " -``` - -Secrets come from `pass` under the `homelab/` prefix. Resolve them inline when deploying. Never hardcode secrets in files. - -## Service YML Conventions - -- All use `version: '3.8'` and Docker Swarm deploy mode -- Each service is pinned to a specific node via `node.hostname` constraint -- All connect to the external overlay network `homelab-net` via `${OVERLAY_NETWORK:-homelab-net}` -- Named volumes for persistence, `on-failure` restart policy with 3 max attempts -- Every service that has a web UI or accepts external connections must publish its ports -- Only include environment variables required to start the service — no optional/nice-to-have vars +Primary workstation (i9-13900KF, 96GB, RTX 4090) is **standalone — not in the Proxmox cluster**. Joins as NATS leaf node for LLM inference. ## Cluster Topology @@ -41,13 +28,75 @@ Secrets come from `pass` under the `homelab/` prefix. Resolve them inline when d | pve-tools | — | 10.10.10.3 | VM on pve, swarm worker | | adder-ghost | — | 10.10.10.20 | VM on adder, swarm worker | +All VMs on vmbr1 (internal, not directly LAN-reachable). A gateway LXC on pve is dual-homed vmbr0+vmbr1 and handles WireGuard + iptables DNAT. + +## VM Allocation (Node 1 / pve — current plan) + +``` +LXC: gateway 256MB — WireGuard client, iptables DNAT, SSH jump host +VM: infra 8GB — NATS/JetStream, n8n +VM: data 20GB — PostgreSQL (shared: Ghost, Forgejo, all apps) +VM: apps 8GB — Ghost instances, Forgejo +VM: redis 14GB — Redis (own VM required — needs vm.overcommit_memory=1 kernel tuning) +VM: nextcloud 8GB — Nextcloud AIO (manages its own isolated internal Postgres) +``` + +No MariaDB/MySQL anywhere — PostgreSQL only. Redis must be a VM (not LXC) for kernel tuning. +Nextcloud's internal DB must stay isolated — opinionated AIO lifecycle, upgrade migrations must not affect other app DBs. + +## NFS / Database Rules + +- **NFS is NOT for live database files.** Use local NVMe for PostgreSQL, Redis, Neo4j, monerod/zanod data. +- LMDB-based services (monerod, zanod) are especially NFS-hostile — mmap is the architecture. +- PostgreSQL tolerates NFS better (WAL + buffer pool) but still prefer local. +- NAS use: VM backups (PBS encrypted), ISOs, Nextcloud user data, blockchain snapshots only. + +## Deployment + +Active service definitions: `proxmox/services/`. The `services/` directory contains older pre-migration copies. + +**Preferred: use `deploy-stack.sh`** (in `~/private/Knowledge/Homelab/deploy-stack.sh`) — scans the compose file for `${VAR}` references, pulls secrets from `pass` or KeePassXC, writes an ephemeral chmod-600 `.env`, deploys, then nukes `.env` on exit via trap. + +```bash +# Deploy using pass backend (default) +./deploy-stack.sh proxmox/services/.yml + +# Deploy to remote swarm manager +./deploy-stack.sh proxmox/services/.yml --host pve-postgres + +# Remove a stack +ssh pve-postgres "docker stack rm " +``` + +Secrets live in `pass` under `homelab/`. Never hardcode secrets in files. + +## Service YML Conventions + +- All use `version: '3.8'` and Docker Swarm deploy mode +- Each service pinned to a specific node via `node.hostname` constraint +- All connect to external overlay network `homelab-net` via `${OVERLAY_NETWORK:-homelab-net}` +- Named volumes for persistence, `on-failure` restart policy with 3 max attempts +- Every service with a web UI or external connections must publish its ports +- Only include environment variables required to start the service — no optional/nice-to-have vars + ## SSH Access Use SSH config aliases (`pve-postgres`, `pve-tools`, `adder-ghost`, `pve`, `adder`, `game`). ProxyJump is configured in `~/.ssh/config`. Never manually hop through intermediate nodes. +## External Infrastructure + +- **BuyVM (Luxembourg):** VPS + 1TB block storage, $7/month, Monero payment. Runs: Peertube, Ghost+BTCPay, Nextcloud, Jitsi, Caddy, WireGuard server. +- **Hetzner AX102 (pending):** Bare metal Proxmox. One VM runs Caddy + WireGuard server — terminates TLS for public subdomains, tunnels traffic to home cluster via WireGuard. +- **Njalla domains:** Accepts Monero, Swedish jurisdiction. + +## Planned Services + +NATS JetStream (3-replica cluster service), PostgreSQL, Redis, Neo4j, FusionAuth, Authentik, n8n, Nextcloud, Garage (object storage), Ghost, monerod, monero-wallet-rpc, zanod, Peertube, BTCPay, Jitsi, Caddy, WireGuard. + ## Key Files - `proxmox/services/*.yml` — Active swarm stack definitions - `proxmox/services/nats.conf` — NATS server config (JetStream, websocket, monitoring) - `proxmox/services/01-init.sql` — Postgres init script (creates users/databases) - `proxmox/post_init_node.org` — Fresh Proxmox node setup steps +- `~/private/Knowledge/Homelab/deploy-stack.sh` — Preferred deployment script diff --git a/proxmox/services/authentik.yml b/proxmox/services/authentik.yml index e01bb44..986630b 100644 --- a/proxmox/services/authentik.yml +++ b/proxmox/services/authentik.yml @@ -20,8 +20,14 @@ services: AUTHENTIK_POSTGRESQL__USER: authentik_user AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_DB_PASSWORD} ports: - - "9000:9000" - - "9443:9443" + - target: 9000 + published: 9000 + protocol: tcp + mode: host + - target: 9443 + published: 9443 + protocol: tcp + mode: host volumes: - authentik_media:/media - authentik_templates:/templates diff --git a/proxmox/services/ghost.yml b/proxmox/services/ghost.yml index 4388f94..0997529 100644 --- a/proxmox/services/ghost.yml +++ b/proxmox/services/ghost.yml @@ -19,7 +19,10 @@ services: database__connection__database: ghost1_db url: ${GHOST1_URL:-http://localhost:2368} ports: - - "2368:2368" + - target: 2368 + published: 2368 + protocol: tcp + mode: host volumes: - ghost1_data:/var/lib/ghost/content networks: @@ -46,7 +49,10 @@ services: url: ${GHOST2_URL:-http://localhost:2369} server__port: 2369 ports: - - "2369:2369" + - target: 2369 + published: 2369 + protocol: tcp + mode: host volumes: - ghost2_data:/var/lib/ghost/content networks: @@ -73,7 +79,10 @@ services: url: ${GHOST3_URL:-http://localhost:2370} server__port: 2370 ports: - - "2370:2370" + - target: 2370 + published: 2370 + protocol: tcp + mode: host volumes: - ghost3_data:/var/lib/ghost/content networks: diff --git a/proxmox/services/mariadb.yml b/proxmox/services/mariadb.yml index 0142852..256426a 100644 --- a/proxmox/services/mariadb.yml +++ b/proxmox/services/mariadb.yml @@ -11,7 +11,10 @@ services: environment: MYSQL_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD} ports: - - "3306:3306" + - target: 3306 + published: 3306 + protocol: tcp + mode: host volumes: - mariadb_data:/var/lib/mysql networks: diff --git a/proxmox/services/n8n.yml b/proxmox/services/n8n.yml index ced0604..aeb2562 100644 --- a/proxmox/services/n8n.yml +++ b/proxmox/services/n8n.yml @@ -16,7 +16,10 @@ services: DB_POSTGRESDB_USER: n8n_user DB_POSTGRESDB_PASSWORD: ${N8N_DB_PASSWORD} ports: - - "5678:5678" + - target: 5678 + published: 5678 + protocol: tcp + mode: host volumes: - n8n_data:/home/node/.n8n networks: diff --git a/proxmox/services/nats.yml b/proxmox/services/nats.yml index 9a856a8..dc9c3e2 100644 --- a/proxmox/services/nats.yml +++ b/proxmox/services/nats.yml @@ -13,9 +13,18 @@ services: command: - -c=/etc/nats/nats.conf ports: - - "4222:4222" - - "8080:8080" - - "8223:8222" + - target: 4222 + published: 4222 + protocol: tcp + mode: host + - target: 8080 + published: 8080 + protocol: tcp + mode: host + - target: 8222 + published: 8223 + protocol: tcp + mode: host volumes: - nats_data:/data configs: diff --git a/proxmox/services/vaultwarden.yml b/proxmox/services/vaultwarden.yml index 4d27825..3242d3e 100644 --- a/proxmox/services/vaultwarden.yml +++ b/proxmox/services/vaultwarden.yml @@ -16,8 +16,12 @@ services: INVITATIONS_ALLOWED: "true" SHOW_PASSWORD_HINT: "false" ROCKET_PORT: 8222 + ADMIN_TOKEN: ${VAULT_ADMIN_TOKEN} ports: - - "8222:8222" + - target: 8222 + published: 8222 + protocol: tcp + mode: host volumes: - vaultwarden_data:/data networks: