diff --git a/CLAUDE.md b/CLAUDE.md index 09fbd88..cf06b97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ hf-pdocker-01 (Manager, 23GB RAM, i7-7700T, NVMe, HD 630 GPU) -- RESTORED (2026- ├── [COMPOSE] Mosquitto (:1883, :8883) ├── [COMPOSE] Synapse/Matrix (:8008) + synapse-db (postgres) + wellknown (:8010) ├── [COMPOSE] Guacamole (:8199) + guacd + guacamole-db (mysql, LOCAL /var/lib/guacamole-db) -├── [COMPOSE] AI Agents (margaret:18790, spirit:18791, clippy:18792, bob:18800) +├── [COMPOSE] AI Agents (margaret:18790, spirit:18795 [infra ops], clippy:18800, bob:18810, travis:18815) ├── [COMPOSE] Traefik (reverse proxy, ports 80/443/8090) ├── [COMPOSE] AdGuard Home (:3000, :53) ├── [COMPOSE] Monitoring (Grafana :3001, Prometheus :9095, Alertmanager :9093, InfluxDB :8086, Blackbox :9115, matrix-receiver :9094) @@ -170,7 +170,7 @@ compose/ # Standalone Compose definitions (deployed per-node) - InfluxDB: :8086 - Blackbox Exporter: :9115 - matrix-alertmanager-receiver: :9094 -- AI Agents (margaret :18790, spirit :18791, clippy :18792, bob :18800) -- loopback only +- AI Agents (margaret :18790, spirit :18795 [infra ops — monitors #notifications, auto-remediates alerts], clippy :18800, bob :18810, travis :18815) -- LAN-accessible (bind=lan, 0.0.0.0) - Farm Manager: :8888 (dashboard + API) - Farm Agent: :8889 @@ -209,6 +209,7 @@ Port conflicts already resolved: - Port 3000: AdGuard Home (Gitea moved to 3003) - Port 3001: Grafana (Semaphore moved to 3004) - Port 5000: Frigate uses host networking (registry container stopped) +- AI Agent ports re-spaced (2026-03-15): openclaw 2026.3.13 uses gateway+2 and gateway+3 for browser/chrome-relay. Agents now spaced 5 apart: margaret=18790, spirit=18795, clippy=18800, bob=18810, travis=18815 ## Critical Configuration Preservation diff --git a/compose/hf-pdocker-01/ai-agents/docker-compose.yml b/compose/hf-pdocker-01/ai-agents/docker-compose.yml new file mode 100644 index 0000000..56b3607 --- /dev/null +++ b/compose/hf-pdocker-01/ai-agents/docker-compose.yml @@ -0,0 +1,125 @@ +services: + margaret: + image: margaret-openclaw:latest + container_name: margaret + restart: unless-stopped + network_mode: host + user: "0:0" + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.openclaw + - OPENCLAW_CONFIG_PATH=/home/glyptic/.openclaw/openclaw.json + volumes: + - /mnt/infrastructure/margaret:/home/glyptic/.openclaw + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 2G + memswap_limit: 2G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + spirit: + image: spirit-ops:latest + container_name: spirit + restart: unless-stopped + network_mode: host + user: "0:0" + labels: + - com.centurylinklabs.watchtower.enable=false + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - WIKIJS_API_KEY=${WIKIJS_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.spirit + - OPENCLAW_CONFIG_PATH=/home/glyptic/.spirit/openclaw.json + volumes: + - /mnt/infrastructure/spirit:/home/glyptic/.spirit + - /var/run/docker.sock:/var/run/docker.sock + - /mnt/infrastructure/spirit/ssh:/root/.ssh:ro + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 3G + memswap_limit: 3G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + bob: + image: margaret-openclaw:latest + container_name: bob + restart: unless-stopped + network_mode: host + user: "0:0" + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.bob + - OPENCLAW_CONFIG_PATH=/home/glyptic/.bob/openclaw.json + volumes: + - /mnt/infrastructure/bob:/home/glyptic/.bob + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 2G + memswap_limit: 2G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + clippy: + image: margaret-openclaw:latest + container_name: clippy + restart: unless-stopped + network_mode: host + user: "0:0" + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.clippy + - OPENCLAW_CONFIG_PATH=/home/glyptic/.clippy/openclaw.json + volumes: + - /mnt/infrastructure/clippy:/home/glyptic/.clippy + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 2G + memswap_limit: 2G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + travis: + image: travis-nemoclaw:latest + container_name: travis + restart: unless-stopped + network_mode: host + user: "0:0" + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.travis + - OPENCLAW_CONFIG_PATH=/home/glyptic/.travis/openclaw.json + volumes: + - /mnt/infrastructure/travis:/home/glyptic/.travis + command: ["sh", "-c", "mkdir -p /home/glyptic && openclaw plugins install /opt/nemoclaw 2>/dev/null; node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 2G + memswap_limit: 2G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" diff --git a/docs/superpowers/plans/2026-03-19-spirit-infra-ops.md b/docs/superpowers/plans/2026-03-19-spirit-infra-ops.md new file mode 100644 index 0000000..14c7204 --- /dev/null +++ b/docs/superpowers/plans/2026-03-19-spirit-infra-ops.md @@ -0,0 +1,724 @@ +# Spirit Infrastructure Ops Agent Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Repurpose the idle Spirit OpenClaw agent as an autonomous infrastructure ops agent that monitors Matrix alerts and takes corrective action. + +**Architecture:** Spirit runs in a custom Docker image with ops tooling (docker-cli, SSH), monitors the #notifications Matrix room for alerts from @alertbot, investigates issues via Docker socket (local) and SSH (remote nodes), takes corrective action, reports to the room, and documents incidents in Wiki.js. + +**Tech Stack:** Docker, OpenClaw, Matrix, SSH, Wiki.js GraphQL API + +**Spec:** `docs/superpowers/specs/2026-03-18-spirit-infra-ops-design.md` + +--- + +## Chunk 1: Image, SSH, and Compose + +### Task 1: Build spirit-ops Docker image on hf-pdocker-01 + +**Context:** Spirit currently uses `margaret-openclaw:latest` (node:22-bookworm-slim + openclaw). We need a new image that adds Docker CLI, SSH client, and diagnostic tools. Build on hf-pdocker-01 since that's where agents run. + +**Files:** +- Create: `/tmp/spirit-ops-build/Dockerfile` (on hf-pdocker-01 via docker-swarm-ops agent) + +- [ ] **Step 1: Create the Dockerfile on hf-pdocker-01** + +Use the docker-swarm-ops agent to SSH to hf-pdocker-01 and create the build directory and Dockerfile: + +```dockerfile +FROM margaret-openclaw:latest + +# Copy Docker CLI from official image (no daemon needed — uses mounted socket) +COPY --from=docker:cli /usr/local/bin/docker /usr/local/bin/docker + +# Install ops tooling +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssh-client \ + curl \ + jq \ + procps \ + net-tools \ + iputils-ping \ + dnsutils \ + && rm -rf /var/lib/apt/lists/* +``` + +Commands: +```bash +ssh glyptic@192.168.86.192 "mkdir -p /tmp/spirit-ops-build" +# SCP or write the Dockerfile +ssh glyptic@192.168.86.192 "cat > /tmp/spirit-ops-build/Dockerfile << 'DOCKERFILE' +FROM margaret-openclaw:latest + +COPY --from=docker:cli /usr/local/bin/docker /usr/local/bin/docker + +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssh-client \ + curl \ + jq \ + procps \ + net-tools \ + iputils-ping \ + dnsutils \ + && rm -rf /var/lib/apt/lists/* +DOCKERFILE" +``` + +- [ ] **Step 2: Build the image** + +```bash +ssh glyptic@192.168.86.192 "cd /tmp/spirit-ops-build && docker build --no-cache -t spirit-ops:latest ." +``` + +Expected: Image builds successfully. Docker CLI is copied as a static binary from the official `docker:cli` image. The existing OpenClaw + Matrix plugin from the base image is preserved. + +- [ ] **Step 3: Verify the image has the required tools** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm spirit-ops:latest sh -c 'docker --version && ssh -V && curl --version | head -1 && jq --version && ps --version && ping -c1 127.0.0.1 > /dev/null && echo ALL OK'" +``` + +Expected: All tool versions print, ending with `ALL OK`. + +--- + +### Task 2: Generate SSH keypair and deploy to nodes + +**Context:** Spirit needs passwordless SSH access to all 3 nodes as user `glyptic`. Keys are stored on NAS at `/mnt/infrastructure/spirit/ssh/` and mounted into the container at `/root/.ssh`. We pre-populate `known_hosts` with real host keys (no StrictHostKeyChecking=no). + +- [ ] **Step 1: Create the ssh directory on NAS** + +```bash +ssh glyptic@192.168.86.192 "sudo mkdir -p /mnt/infrastructure/spirit/ssh && sudo chown root:root /mnt/infrastructure/spirit/ssh && sudo chmod 700 /mnt/infrastructure/spirit/ssh" +``` + +Note: NFS root_squash may prevent chown. Use the busybox workaround if needed: +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data busybox sh -c 'mkdir -p /data/ssh && chmod 700 /data/ssh'" +``` + +- [ ] **Step 2: Generate ED25519 keypair** + +Generate the key on hf-pdocker-01 and copy to the NAS SSH directory: + +```bash +ssh glyptic@192.168.86.192 "ssh-keygen -t ed25519 -f /tmp/spirit-ssh-key -N '' -C spirit-ops@homelab && sudo cp /tmp/spirit-ssh-key /mnt/infrastructure/spirit/ssh/id_ed25519 && sudo cp /tmp/spirit-ssh-key.pub /mnt/infrastructure/spirit/ssh/id_ed25519.pub && sudo chmod 600 /mnt/infrastructure/spirit/ssh/id_ed25519 && sudo chmod 644 /mnt/infrastructure/spirit/ssh/id_ed25519.pub && rm /tmp/spirit-ssh-key /tmp/spirit-ssh-key.pub" +``` + +Expected: `/mnt/infrastructure/spirit/ssh/id_ed25519` and `.pub` files exist with correct permissions. + +- [ ] **Step 3: Create SSH config file** + +```bash +ssh glyptic@192.168.86.192 "sudo tee /mnt/infrastructure/spirit/ssh/config > /dev/null << 'EOF' +Host hf-pdocker-01 + HostName 192.168.86.192 + User glyptic + IdentityFile /root/.ssh/id_ed25519 + +Host hf-pdocker-02 + HostName 192.168.86.100 + User glyptic + IdentityFile /root/.ssh/id_ed25519 + +Host bart + HostName 192.168.86.167 + User glyptic + IdentityFile /root/.ssh/id_ed25519 + +Host * + ServerAliveInterval 30 + ServerAliveCountMax 3 + UserKnownHostsFile /root/.ssh/known_hosts +EOF" +ssh glyptic@192.168.86.192 "sudo chmod 600 /mnt/infrastructure/spirit/ssh/config" +``` + +- [ ] **Step 4: Collect host keys and create known_hosts** + +```bash +ssh glyptic@192.168.86.192 "ssh-keyscan -H 192.168.86.192 192.168.86.100 192.168.86.167 2>/dev/null | sudo tee /mnt/infrastructure/spirit/ssh/known_hosts > /dev/null && sudo chmod 644 /mnt/infrastructure/spirit/ssh/known_hosts" +``` + +Expected: `known_hosts` file contains SSH host keys for all 3 nodes. + +- [ ] **Step 5: Deploy public key to hf-pdocker-02 and bart** + +Read the public key first: +```bash +ssh glyptic@192.168.86.192 "cat /mnt/infrastructure/spirit/ssh/id_ed25519.pub" +``` + +Then append to authorized_keys on each remote node: +```bash +# hf-pdocker-02 +ssh glyptic@192.168.86.100 "mkdir -p ~/.ssh && chmod 700 ~/.ssh" +ssh glyptic@192.168.86.192 "cat /mnt/infrastructure/spirit/ssh/id_ed25519.pub" | ssh glyptic@192.168.86.100 "cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" + +# bart +ssh glyptic@192.168.86.167 "mkdir -p ~/.ssh && chmod 700 ~/.ssh" +ssh glyptic@192.168.86.192 "cat /mnt/infrastructure/spirit/ssh/id_ed25519.pub" | ssh glyptic@192.168.86.167 "cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" +``` + +Also deploy to hf-pdocker-01 itself (Spirit may SSH to localhost): +```bash +ssh glyptic@192.168.86.192 "cat /mnt/infrastructure/spirit/ssh/id_ed25519.pub >> ~/.ssh/authorized_keys" +``` + +- [ ] **Step 6: Verify SSH connectivity from a test container** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -v /mnt/infrastructure/spirit/ssh:/root/.ssh:ro spirit-ops:latest sh -c 'ssh -o BatchMode=yes glyptic@192.168.86.100 hostname && ssh -o BatchMode=yes glyptic@192.168.86.167 hostname && echo SSH OK'" +``` + +Expected: Prints `hf-pdocker-02`, `bart`, then `SSH OK`. + +--- + +### Task 3: Update compose file + +**Context:** Modify the `spirit` service in the ai-agents compose file to use the new image, mount docker socket and SSH keys, increase memory, and add the Wiki.js API key env var. + +**Files:** +- Modify: `compose/hf-pdocker-01/ai-agents/docker-compose.yml:26-48` + +- [ ] **Step 1: Update the spirit service in docker-compose.yml** + +Change the spirit service block (lines 26-48) to: + +```yaml + spirit: + image: spirit-ops:latest + container_name: spirit + restart: unless-stopped + network_mode: host + user: "0:0" + labels: + - com.centurylinklabs.watchtower.enable=false + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - WIKIJS_API_KEY=${WIKIJS_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.spirit + - OPENCLAW_CONFIG_PATH=/home/glyptic/.spirit/openclaw.json + volumes: + - /mnt/infrastructure/spirit:/home/glyptic/.spirit + - /var/run/docker.sock:/var/run/docker.sock + - /mnt/infrastructure/spirit/ssh:/root/.ssh:ro + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + mem_limit: 3G + memswap_limit: 3G + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" +``` + +- [ ] **Step 2: Add WIKIJS_API_KEY to the .env file on NAS** + +The .env file is at `/mnt/docker-data/compose/hf-pdocker-01/ai-agents/.env` on the NAS. Retrieve the Wiki.js API key from the documentation-keeper agent config or Wiki.js admin panel, then append: + +```bash +ssh glyptic@192.168.86.192 "echo 'WIKIJS_API_KEY=' | sudo tee -a /mnt/docker-data/compose/hf-pdocker-01/ai-agents/.env > /dev/null" +``` + +The JWT token is the long-lived API key from Wiki.js (Administration > API Access). It's already used by the documentation-keeper agent. + +--- + +## Chunk 2: Workspace Identity Files and Model Config + +### Task 4: Update Spirit's openclaw.json model to Claude + +**Context:** Spirit currently uses `ollama/qwen3.5` as primary model. For an infrastructure ops agent that needs strong reasoning and tool use, Claude is required. Update the primary model to `anthropic/claude-sonnet-4-5-20250514` (good balance of speed and capability for ops tasks — opus is overkill for container restarts). + +- [ ] **Step 1: Update the model in openclaw.json** + +Use the NFS write workaround since openclaw.json is root-owned: + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data alpine sh -c \"apk add --no-cache jq > /dev/null 2>&1 && jq '.models.primary = \"anthropic/claude-sonnet-4-5-20250514\"' /data/openclaw.json > /data/openclaw.json.tmp && mv /data/openclaw.json.tmp /data/openclaw.json\"" +``` + +- [ ] **Step 2: Verify the change** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data busybox cat /data/openclaw.json" | grep -o '"primary":"[^"]*"' +``` + +Expected: `"primary":"anthropic/claude-sonnet-4-5-20250514"` + +--- + +### Task 5: Write SOUL.md workspace identity file + +**Context:** Spirit's current SOUL.md is stale (references old hostname "Peggy"). Replace with infrastructure ops identity. The file is glyptic-owned so may be writable directly, but use the NFS workaround if needed. + +- [ ] **Step 1: Write SOUL.md to NAS** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data busybox sh -c 'cat > /data/workspace/SOUL.md << \"SOULEOF\" +# Spirit — Infrastructure Operations Agent + +You are Spirit, an autonomous infrastructure operations agent for a homelab cluster. You monitor the #notifications Matrix room for alerts from @alertbot and take corrective action to keep services running. + +## Core Principles + +1. **Service availability is your top priority.** When an alert fires, investigate immediately and take action to restore service. +2. **Always report what you find and what you do.** Post your investigation results and any actions taken back to the #notifications Matrix room so the owner can see what happened. +3. **Document everything.** After taking corrective action, update the relevant Wiki.js page with an incident entry. +4. **Respect the guardrails.** Some services are protected and must not be restarted. Rate-limit your actions. Escalate when remediation fails. +5. **Data safety matters.** Never delete volumes, drop databases, or run destructive commands. Prefer restarts and cleanup over risky operations. + +## How You Work + +- You receive alerts as Matrix messages from @alertbot:hobbs.farm in the #notifications:hobbs.farm room +- You have Docker CLI access on hf-pdocker-01 (local socket) and SSH access to hf-pdocker-02 and bart +- You can run any diagnostic or remediation command on any node +- You post your findings and actions back to the same Matrix room +- You update Wiki.js documentation after taking corrective action +SOULEOF'" +``` + +--- + +### Task 6: Write IDENTITY.md workspace identity file + +**Context:** This is the comprehensive operational knowledge file. It tells Spirit everything about the cluster, alerts, remediation procedures, and guardrails. + +- [ ] **Step 1: Write IDENTITY.md to NAS** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data busybox sh -c 'cat > /data/workspace/IDENTITY.md << \"IDEOF\" +# Infrastructure Operations Reference + +## Cluster Architecture + +### Nodes + +| Node | IP | RAM | Role | Access Method | +|------|-----|-----|------|---------------| +| hf-pdocker-01 | 192.168.86.192 | 23GB | Swarm Manager, primary services | Local Docker socket | +| hf-pdocker-02 | 192.168.86.100 | 7.6GB | Dev tools, utilities | SSH as glyptic | +| bart | 192.168.86.167 | 15GB | Media stack, Onyx | SSH as glyptic | +| Synology NAS | 192.168.86.30 | - | NFS storage for all services | Not directly accessible | + +### Services by Node + +**hf-pdocker-01 (192.168.86.192):** +- Frigate NVR (:5000) — 13 cameras, OpenVINO GPU detector +- Frigate-Notify v2 (:5199) — MQTT→AI→Matrix notifications +- clip-service (:5200) — CLIP embeddings +- Open WebUI (:3010) — AI chat UI +- Home Assistant (:8123) — home automation +- ESPHome (:6052) — ESP device management +- Mosquitto (:1883, :8883) — MQTT broker +- Matrix/Synapse (:8008) — chat server +- Well-known (:8010) — Matrix federation +- Guacamole (:8199) — remote desktop gateway +- AI Agents: margaret(:18790), spirit(:18795/you), clippy(:18800), bob(:18810), travis(:18815) +- Traefik (:80, :443, :8090) — reverse proxy +- AdGuard Home (:3000, :53) — DNS +- Grafana (:3001), Prometheus (:9095), Alertmanager (:9093) +- InfluxDB (:8086), Blackbox Exporter (:9115) +- matrix-alertmanager-receiver (:9094) — alert→Matrix bridge +- Farm Manager (:8888), Farm Agent (:8889) + +**hf-pdocker-02 (192.168.86.100):** +- Unifi (:8443) — network controller +- Gitea (:3003) — git server +- Semaphore (:3004) — Ansible UI +- Wiki.js (:3002) — documentation +- Vaultwarden (:26226) — password manager +- Actual Budget (:5006) — finance +- MCPO (:8001), Trivy (:4954) +- SearXNG (:8888) — meta-search +- Unstructured API (:8000) — document parsing +- n8n + n8n-db — workflow automation +- NetClaw/Aaron (:1179) — AI network agent +- Farm Agent (:8889) +- Watchtower — auto-updates + +**bart (192.168.86.167):** +- Jellyfin (:8096) — media server +- Sonarr (:8989), Radarr (:7878) — media management +- SABnzbd (:8082), NZBHydra2 (:5076) — downloads +- Jellyseerr (:5055) — media requests +- OpenVAS (:9392) — vulnerability scanner +- Onyx (:3005) — AI document search/RAG +- Farm Agent (:8889) +- Watchtower — auto-updates + +### Storage + +All service data on NAS via NFS at /mnt/docker-data/ on every node: +- configs/{service}/ — application configs +- databases/{service}-db/ — database volumes +- compose/{node}/{service}/ — compose files + +### Compose File Locations + +To manage services, the compose files are at: +- hf-pdocker-01: /mnt/docker-data/compose/hf-pdocker-01/{service}/docker-compose.yml +- hf-pdocker-02: /mnt/docker-data/compose/hf-pdocker-02/{service}/docker-compose.yml +- bart: /mnt/docker-data/compose/bart/{service}/docker-compose.yml + +To restart a compose service on a remote node: +``` +ssh glyptic@{node-ip} "cd /mnt/docker-data/compose/{node}/{service} && docker compose up -d" +``` + +## Commands + +### Local (hf-pdocker-01 — Docker socket) +```bash +docker ps # list running containers +docker ps -a # list all containers including stopped +docker logs --tail 100 # view container logs +docker restart # restart a container +docker stats --no-stream # resource usage snapshot +docker system df # disk usage by Docker +docker system prune -f # clean up unused images/containers (NOT volumes) +docker compose -f /path/to/docker-compose.yml up -d # redeploy a service +``` + +### Remote (hf-pdocker-02, bart — via SSH) +```bash +ssh glyptic@ "docker ps" +ssh glyptic@ "docker logs --tail 100" +ssh glyptic@ "docker restart " +ssh glyptic@ "df -h" +ssh glyptic@ "free -m" +ssh glyptic@ "cd /mnt/docker-data/compose// && docker compose up -d" +``` + +### System Diagnostics +```bash +df -h # disk usage +free -m # memory usage +top -bn1 | head -20 # CPU/process snapshot +uptime # load averages +mount | grep nfs # NFS mount status +ping -c3 192.168.86.30 # NAS reachability +``` + +## Safety Guardrails + +### Protected Services — NEVER restart these without explicit owner request +- **Monitoring pipeline**: prometheus, alertmanager, matrix-alertmanager-receiver, grafana, blackbox-exporter +- **Network infrastructure**: traefik, adguard +- **AI agents**: margaret, spirit (yourself), bob, clippy, travis +- **All database containers**: synapse-db, guacamole-db, wikijs-db, gitea-db, semaphore-db, unifi-db, n8n-db, any container ending in -db + +If a protected service is down, INVESTIGATE and REPORT but do NOT restart. Tell the owner what you found. + +### Rate Limiting +- Do NOT restart the same container more than once per 30 minutes +- Maximum 5 remediation actions per hour across all containers +- After 2 failed remediation attempts for the same alert, STOP and escalate to the owner with full diagnostics + +### Rollback Awareness +- After restarting a container, wait 5 minutes. If the same alert fires again, do NOT restart again — report the persistent failure with full logs +- NEVER run: docker system prune --volumes, docker volume rm, rm -rf on data directories +- docker system prune -f (without --volumes) IS allowed for disk cleanup + +## Alert Response Procedures + +### Response Flow +1. Parse the alert message from @alertbot — extract: alert name, severity, instance, description +2. Investigate — run relevant diagnostic commands +3. Remediate — take corrective action (if the service is not protected and rate limits allow) +4. Report — post findings and actions to #notifications room +5. Document — update the relevant Wiki.js page + +### Alert-Specific Procedures + +| Alert | Severity | What to Check | What to Do | +|-------|----------|---------------|------------| +| ContainerDown | warning | docker ps -a, docker logs | docker restart or docker compose up -d | +| HighDiskUsage | critical | df -h, docker system df | docker system prune -f, identify large files | +| DiskSpaceWarning | warning | df -h, docker system df | Report findings, suggest cleanup | +| HighMemoryUsage | warning | free -m, docker stats --no-stream | Identify top memory consumer, restart if non-protected | +| CriticalMemoryUsage | critical | free -m, docker stats --no-stream | Restart heaviest non-protected container | +| HighCPUUsage | warning | top -bn1, docker stats --no-stream | Identify CPU hog, report | +| CriticalCPUUsage | critical | top -bn1, docker stats --no-stream | Restart CPU hog if non-protected | +| HighSystemLoad | warning | uptime, top -bn1, docker stats | Report load source | +| NFSMountDown | critical | mount, ping 192.168.86.30 | Try sudo mount -a, report if NAS unreachable | +| NASDiskSpaceLow | warning | df -h /mnt/docker-data | Report, identify large consumers | +| SlowHTTPResponse | warning | curl timing, docker logs | Restart slow service if non-protected | +| FrigateHighMemory | warning | docker stats frigate | docker restart frigate | +| ServiceHighMemory | warning | docker stats | Restart container if non-protected | +| ServerDown | critical | ping, SSH attempt | Report — cannot fix hardware | +| ServerRebooted | info | uptime, docker ps | Report, verify all containers running | +| HighSwapUsage | warning | free -m, docker stats | Identify swap contributor | +| CriticalSwapUsage | critical | free -m, docker stats | Restart heaviest swap user if non-protected | +| PrometheusTargetDown | warning | curl the target, docker ps | Restart target if down and non-protected | +| Watchdog | info | N/A | Ignore — dead-man switch | +| UnifiDeviceDown | warning | Check Unifi controller UI | Report — network device issue | + +### Resolved Alerts +When you see a RESOLVED alert from @alertbot, acknowledge it briefly. No investigation needed. + +### Unknown Alerts +For any alert type not listed above: investigate and report findings, but do NOT take remediation action. + +## Wiki.js Documentation + +After taking any corrective action, update the relevant Wiki.js page. + +**API endpoint**: http://192.168.86.100:3002/graphql +**Auth**: Bearer token from WIKIJS_API_KEY environment variable + +**Key pages:** +- infrastructure/overview (ID 118) +- infrastructure/service-access (ID 119) +- infrastructure/maintenance (ID 120) +- infrastructure/alerting (ID 121) + +**To read a page:** +```bash +curl -s http://192.168.86.100:3002/graphql \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $WIKIJS_API_KEY" \ + -d '"'"'{"query":"{ pages { single(id: 120) { id title content } } }"}'"'"' +``` + +**To update a page** (append incident entry to existing content): +1. Read current page content +2. Append your incident entry at the bottom +3. Update the page with the full new content + +Incident entry format: +```markdown +### YYYY-MM-DD HH:MM — [Alert Name] ([severity]) +- **Node**: [affected node] +- **Service**: [affected service] +- **Root cause**: [what you found] +- **Action taken**: [what you did] +- **Outcome**: [result — resolved/escalated/persistent] +``` +IDEOF'" +``` + +Note: This is a large heredoc. If it fails due to shell escaping issues, use the base64 approach instead: +1. Write IDENTITY.md content to a local temp file +2. Base64 encode and pipe to the target: +```bash +base64 /tmp/IDENTITY.md | ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data -i busybox sh -c 'base64 -d > /data/workspace/IDENTITY.md'" +``` + +Or write locally and SCP directly: +```bash +scp /tmp/IDENTITY.md glyptic@192.168.86.192:/tmp/IDENTITY.md +ssh glyptic@192.168.86.192 "sudo cp /tmp/IDENTITY.md /mnt/infrastructure/spirit/workspace/IDENTITY.md" +``` + +--- + +### Task 7: Write USER.md workspace identity file + +- [ ] **Step 1: Write USER.md to NAS** + +```bash +ssh glyptic@192.168.86.192 "docker run --rm -u 0:0 -v /mnt/infrastructure/spirit:/data busybox sh -c 'cat > /data/workspace/USER.md << \"USEREOF\" +# Owner: Jeremy + +- Homelab operator, full infrastructure access +- Prefers autonomous action — fix things first, report after +- Full trust for corrective actions within the safety guardrails +- Timezone: US Central (CST/CDT) +- Reachable via Matrix: @jeremy:hobbs.farm +- If something needs human attention, post in #notifications — Jeremy monitors it +USEREOF'" +``` + +--- + +## Chunk 3: Deploy and Verify + +### Task 8: Deploy Spirit with new configuration + +**Context:** SCP the updated compose file to hf-pdocker-01, then redeploy only the Spirit container. Other agents (margaret, clippy, bob, travis) remain untouched. + +- [ ] **Step 1: SCP the updated compose file to the NAS** + +```bash +scp compose/hf-pdocker-01/ai-agents/docker-compose.yml glyptic@192.168.86.192:/mnt/docker-data/compose/hf-pdocker-01/ai-agents/docker-compose.yml +``` + +- [ ] **Step 2: Stop only the Spirit container** + +```bash +ssh glyptic@192.168.86.192 "docker stop spirit && docker rm spirit" +``` + +- [ ] **Step 3: Redeploy Spirit using docker compose** + +```bash +ssh glyptic@192.168.86.192 "cd /mnt/docker-data/compose/hf-pdocker-01/ai-agents && docker compose up -d spirit" +``` + +Expected: Only the `spirit` container is recreated. Other agents remain running. + +- [ ] **Step 4: Verify Spirit container is running with correct image and mounts** + +```bash +ssh glyptic@192.168.86.192 "docker inspect spirit --format '{{.Config.Image}} {{.HostConfig.Memory}} {{range .Mounts}}{{.Source}}->{{.Destination}} {{end}}'" +``` + +Expected: Image is `spirit-ops:latest`, memory is 3221225472 (3GB), mounts include `/var/run/docker.sock` and `/mnt/infrastructure/spirit/ssh`. + +- [ ] **Step 5: Verify Docker socket works inside Spirit** + +```bash +ssh glyptic@192.168.86.192 "docker exec spirit docker ps --format 'table {{.Names}}\t{{.Status}}' | head -5" +``` + +Expected: Lists running containers on hf-pdocker-01. + +- [ ] **Step 6: Verify SSH works inside Spirit** + +```bash +ssh glyptic@192.168.86.192 "docker exec spirit ssh -o BatchMode=yes glyptic@192.168.86.100 hostname" +ssh glyptic@192.168.86.192 "docker exec spirit ssh -o BatchMode=yes glyptic@192.168.86.167 hostname" +``` + +Expected: Prints `hf-pdocker-02` and `bart`. + +- [ ] **Step 7: Wait for Spirit's Matrix plugin to initialize (90-120 seconds)** + +```bash +ssh glyptic@192.168.86.192 "docker logs spirit --tail 20" +``` + +Expected: After 90-120 seconds, logs show Matrix plugin connected. No output during crypto init is normal. + +--- + +### Task 9: Invite Spirit to #notifications room + +**Context:** Use the Synapse admin API to invite @spirit:hobbs.farm to the notifications room. Spirit must already be running so the Matrix plugin accepts the invite. + +- [ ] **Step 1: Invite Spirit to the room using Synapse admin API** + +Use Spirit's own admin token (Spirit is a Synapse admin): + +```bash +ssh glyptic@192.168.86.192 "curl -s -X POST 'http://192.168.86.192:8008/_matrix/client/v3/rooms/!jYmqxbzEsvRgKYvwMP:hobbs.farm/invite' -H 'Authorization: Bearer syt_c3Bpcml0_AUBChFVZIEcXzOjUnNHo_0aaMzg' -H 'Content-Type: application/json' -d '{\"user_id\": \"@spirit:hobbs.farm\"}'" +``` + +Note: If Spirit can't invite itself, use the alertbot token or another admin account: +```bash +ssh glyptic@192.168.86.192 "curl -s -X POST 'http://192.168.86.192:8008/_matrix/client/v3/rooms/!jYmqxbzEsvRgKYvwMP:hobbs.farm/invite' -H 'Authorization: Bearer syt_YWxlcnRib3Q_eOVAYjYKWBZxymfdxhPE_0iYlOZ' -H 'Content-Type: application/json' -d '{\"user_id\": \"@spirit:hobbs.farm\"}'" +``` + +Expected: `{}` or `{"room_id":"!jYmqxbzEsvRgKYvwMP:hobbs.farm"}` — success. + +- [ ] **Step 2: Verify Spirit joined the room** + +Wait 10-30 seconds for the OpenClaw Matrix plugin to auto-accept, then check: + +```bash +ssh glyptic@192.168.86.192 "curl -s 'http://192.168.86.192:8008/_matrix/client/v3/rooms/!jYmqxbzEsvRgKYvwMP:hobbs.farm/joined_members' -H 'Authorization: Bearer syt_c3Bpcml0_AUBChFVZIEcXzOjUnNHo_0aaMzg' | jq '.joined | keys'" +``` + +Expected: List includes `@spirit:hobbs.farm` alongside `@alertbot:hobbs.farm`. + +--- + +### Task 10: End-to-end test with a real alert + +**Context:** Send a test alert through Alertmanager to trigger the full pipeline: Alertmanager → matrix-alertmanager-receiver → @alertbot → #notifications → Spirit sees it and responds. + +- [ ] **Step 1: Send a test alert via Alertmanager API** + +```bash +ssh glyptic@192.168.86.192 "curl -s -X POST http://192.168.86.192:9093/api/v2/alerts -H 'Content-Type: application/json' -d '[{ + \"labels\": { + \"alertname\": \"TestAlert\", + \"severity\": \"info\", + \"instance\": \"test-instance:9999\", + \"job\": \"test\" + }, + \"annotations\": { + \"summary\": \"Test alert for Spirit ops agent verification\", + \"description\": \"This is a test alert. Spirit should acknowledge it and report in the notifications channel.\" + }, + \"generatorURL\": \"http://192.168.86.192:9095\" +}]'" +``` + +Expected: HTTP 200. Alert flows through the pipeline. + +- [ ] **Step 2: Verify Spirit responds in the Matrix room** + +Wait 1-2 minutes, then check recent messages in the room: + +```bash +ssh glyptic@192.168.86.192 "curl -s 'http://192.168.86.192:8008/_matrix/client/v3/rooms/!jYmqxbzEsvRgKYvwMP:hobbs.farm/messages?dir=b&limit=5' -H 'Authorization: Bearer syt_c3Bpcml0_AUBChFVZIEcXzOjUnNHo_0aaMzg' | jq '.chunk[] | {sender: .sender, body: .content.body}'" +``` + +Expected: Messages from both `@alertbot:hobbs.farm` (the alert) and `@spirit:hobbs.farm` (the response/acknowledgement). + +- [ ] **Step 3: Resolve the test alert** + +```bash +ssh glyptic@192.168.86.192 "curl -s -X POST http://192.168.86.192:9093/api/v2/alerts -H 'Content-Type: application/json' -d '[{ + \"labels\": { + \"alertname\": \"TestAlert\", + \"severity\": \"info\", + \"instance\": \"test-instance:9999\", + \"job\": \"test\" + }, + \"annotations\": { + \"summary\": \"Test alert for Spirit ops agent verification\", + \"description\": \"This is a test alert.\" + }, + \"endsAt\": \"2026-03-19T00:00:00Z\", + \"generatorURL\": \"http://192.168.86.192:9095\" +}]'" +``` + +Expected: Alertmanager sends a "resolved" notification. Spirit acknowledges it briefly. + +--- + +### Task 11: Post-deployment updates + +- [ ] **Step 1: Commit compose changes and spec to git** + +```bash +git add compose/hf-pdocker-01/ai-agents/docker-compose.yml +git add docs/superpowers/specs/2026-03-18-spirit-infra-ops-design.md +git add docs/superpowers/plans/2026-03-19-spirit-infra-ops.md +git commit -m "infra: repurpose Spirit as infrastructure ops agent with alert monitoring" +``` + +- [ ] **Step 2: Update CLAUDE.md** + +Update the Spirit entry in CLAUDE.md's service distribution to reflect its new role. Change the AI Agents line or add a note that Spirit is now the infrastructure ops agent monitoring #notifications. + +- [ ] **Step 3: Update MEMORY.md** + +Add a memory entry for Spirit's new role as infrastructure ops agent. + +- [ ] **Step 4: Add Prometheus probe for Spirit** + +Spirit is now a critical infrastructure component. Add `http://192.168.86.192:18795` to the `http-pdocker01` job in `stacks/monitoring/prometheus.yml`, then SCP and hot-reload: + +```bash +# After editing prometheus.yml locally: +scp stacks/monitoring/prometheus.yml glyptic@192.168.86.192:/mnt/docker-data/configs/prometheus/prometheus.yml +ssh glyptic@192.168.86.192 "curl -X POST http://192.168.86.192:9095/-/reload" +``` + +- [ ] **Step 5: Push to Gitea** + +Use the gitea-backup agent to commit and push all changes. + +- [ ] **Step 6: Update Wiki.js** + +Use the documentation-keeper agent to update the infrastructure/alerting wiki page (ID 121) documenting that Spirit now monitors and responds to alerts in #notifications. diff --git a/docs/superpowers/specs/2026-03-18-spirit-infra-ops-design.md b/docs/superpowers/specs/2026-03-18-spirit-infra-ops-design.md new file mode 100644 index 0000000..a2a8d0e --- /dev/null +++ b/docs/superpowers/specs/2026-03-18-spirit-infra-ops-design.md @@ -0,0 +1,191 @@ +# Spirit Infrastructure Ops Agent + +## Overview + +Repurpose the idle Spirit OpenClaw agent (`@spirit:hobbs.farm`) as an autonomous infrastructure operations agent. Spirit joins the `#notifications:hobbs.farm` Matrix room, monitors alerts posted by `@alertbot:hobbs.farm`, investigates issues, takes corrective action, and reports findings back to the room. + +## Architecture + +``` +Prometheus → Alertmanager → @alertbot → #notifications room + ↓ + @spirit reads alert + ↓ + Investigate (docker/SSH) + ↓ + Remediate + report to room +``` + +Spirit has full autonomy to take corrective action, subject to the safety guardrails defined below. + +## Cluster Context + +Spirit must have complete knowledge of the infrastructure: + +- **hf-pdocker-01** (192.168.86.192) — Swarm Manager, 23GB RAM. Runs: Frigate, Frigate-Notify, clip-service, Open WebUI, Home Assistant, ESPHome, Mosquitto, Matrix/Synapse, Guacamole, AI Agents, Traefik, AdGuard, Monitoring stack, Farm Manager (:8888), Farm Agent (:8889). Docker socket available locally. +- **hf-pdocker-02** (192.168.86.100) — 7.6GB RAM. Runs: Unifi, Gitea, Semaphore, Wiki.js, Vaultwarden, Actual Budget, MCPO, Trivy, SearXNG, Unstructured API, n8n, NetClaw, Farm Agent, Watchtower. Access via SSH. +- **bart** (192.168.86.167) — 15GB RAM. Runs: Jellyfin, Sonarr, Radarr, SABnzbd, NZBHydra2, Jellyseerr, OpenVAS, Onyx, Farm Agent, Watchtower. Access via SSH. +- **Synology NAS** (192.168.86.30) — NFS storage at `/mnt/docker-data` on all nodes. + +## Components + +### 1. Custom Docker Image: `spirit-ops:latest` + +Build location: `/tmp/spirit-ops-build/` on hf-pdocker-01. + +Extends `margaret-openclaw:latest` with operational tooling: + +- `docker-cli` (Docker CLI, no daemon — uses mounted socket) +- `openssh-client` (SSH to remote nodes) +- `curl`, `jq` (API calls, JSON parsing) +- `procps`, `net-tools`, `iputils-ping`, `dnsutils` (system diagnostics) + +The Dockerfile installs these packages on top of the existing OpenClaw + Matrix plugin base. + +### 2. Compose Changes + +In `compose/hf-pdocker-01/ai-agents/docker-compose.yml`, modify the `spirit` service: + +- Change image from `margaret-openclaw:latest` to `spirit-ops:latest` +- Add volume mount: `/var/run/docker.sock:/var/run/docker.sock` (local Docker access) +- Add volume mount: `/mnt/infrastructure/spirit/ssh:/root/.ssh:ro` (SSH keys — mounted to `/root/.ssh` since container runs as root) +- Increase `mem_limit` and `memswap_limit` to `3G` (diagnostic commands + alert handling need headroom beyond the 1.5G Node.js allocation) +- Add environment variable `WIKIJS_API_KEY` (the long-lived JWT token for Wiki.js GraphQL API) +- All other settings (ports, network mode) remain unchanged. + +Other agent services (margaret, clippy, bob, travis) are not modified. + +### 3. SSH Key Setup + +- Generate ED25519 keypair on hf-pdocker-01 +- Store private key at `/mnt/infrastructure/spirit/ssh/id_ed25519` (NAS, mounted into container) +- Store public key at `/mnt/infrastructure/spirit/ssh/id_ed25519.pub` +- Create `/mnt/infrastructure/spirit/ssh/config` with: + - `IdentityFile /root/.ssh/id_ed25519` (explicit path) + - Pre-populated `known_hosts` file with actual host keys from all 3 nodes (no `StrictHostKeyChecking=no`) +- Deploy public key to `~glyptic/.ssh/authorized_keys` on hf-pdocker-02 and bart +- SSH user: `glyptic` (has sudo on all nodes) + +### 4. Workspace Identity Files + +Rewrite Spirit's workspace files on NAS at `/mnt/infrastructure/spirit/workspace/`: + +**SOUL.md** — Core identity: +- Spirit is an infrastructure operations agent for a homelab cluster +- Full autonomy to investigate and remediate alerts, subject to safety guardrails +- Always reports actions and findings to the Matrix room +- Prioritizes service availability and data safety + +**IDENTITY.md** — Detailed operational knowledge: +- Complete cluster architecture (nodes, IPs, services, ports) +- Compose file locations on NAS (`/mnt/docker-data/compose/{node}/{service}/`) +- All alert types and remediation procedures (see table below) +- Docker commands for local ops, SSH commands for remote ops +- NFS mount details and known issues (root_squash, database user requirements) +- Safety guardrails and protected services list +- Escalation behavior and rate limiting rules + +**USER.md** — Owner context: +- Jeremy, homelab operator +- Prefers autonomous action with post-action reporting +- Full trust for corrective actions within guardrails + +### 5. Matrix Room Membership + +- Invite `@spirit:hobbs.farm` to `!jYmqxbzEsvRgKYvwMP:hobbs.farm` using Synapse admin API +- Spirit's OpenClaw Matrix plugin auto-accepts room invites +- Spirit begins monitoring all messages in the room +- Spirit responds to alert messages from `@alertbot:hobbs.farm` +- Spirit distinguishes alerts (HTML-formatted messages from @alertbot with severity/color coding) from normal conversation + +## Safety Guardrails + +### Protected Services (never restart without explicit owner request) +- Prometheus, Alertmanager, matrix-alertmanager-receiver (restarting breaks the alerting pipeline) +- Traefik (restarting drops all external access) +- Spirit's own container and other AI agents +- Database containers (*-db, guacamole-db, etc.) + +### Rate Limiting +- **Per-container cooldown**: Do not restart the same container more than once per 30 minutes +- **Global budget**: Maximum 5 remediation actions per hour +- **Escalation**: After 2 failed remediation attempts for the same alert, stop trying and report to the owner with full diagnostics + +### Rollback Awareness +- After restarting a container, monitor for 5 minutes. If the same alert re-fires, do not attempt further restarts — report the persistent failure with logs. +- Never delete data, drop volumes, or run `docker system prune --volumes` +- `docker system prune -f` (without --volumes) is allowed for disk cleanup + +### Command Logging +- Report every command executed and its output to the Matrix room, either inline or as a summary after investigation + +### Wiki Documentation +- After taking any corrective action, Spirit updates the relevant Wiki.js page to document what happened, what was done, and when +- Wiki.js GraphQL API: `http://192.168.86.100:3002/graphql` (internal, from container via SSH or direct HTTP) +- Authentication: Bearer token (long-lived JWT API key, stored as env var `WIKIJS_API_KEY` in Spirit's compose) +- Spirit appends incident entries to the relevant wiki page (e.g., infrastructure/alerting page ID 121, or service-specific pages) +- Documentation format: timestamp, alert name, severity, root cause (if identified), actions taken, outcome +- Key wiki pages: infrastructure/overview (118), service-access (119), maintenance (120), alerting (121) +- Uses GraphQL mutations: `pages { update(id, content, ...) }` to append to existing pages + +## Alert Response Behavior + +When Spirit sees an alert from @alertbot: + +1. **Parse** the alert: extract alert name, severity, instance, description +2. **Investigate**: run relevant diagnostics (docker ps, logs, df, free, curl health endpoints) +3. **Remediate**: take corrective action based on alert type (respecting guardrails) +4. **Report**: post findings and actions taken back to #notifications +5. **Document**: update the relevant Wiki.js page with an incident entry (timestamp, alert, actions taken, outcome) + +### Remediation Procedures (All Alert Types) + +| Alert | Severity | Investigation | Remediation | +|-------|----------|--------------|-------------| +| ContainerDown | warning | `docker ps -a`, `docker logs ` | `docker restart ` or `docker compose up -d` | +| HighDiskUsage | critical | `df -h`, `docker system df` | `docker system prune -f`, clear old logs | +| DiskSpaceWarning | warning | `df -h`, `docker system df` | Report findings, suggest cleanup | +| HighMemoryUsage | warning | `free -m`, `docker stats --no-stream` | Identify memory hog, restart if needed | +| CriticalMemoryUsage | critical | Same as above | Restart heaviest non-protected container, report | +| HighCPUUsage | warning | `top -bn1`, `docker stats --no-stream` | Identify CPU hog, report | +| CriticalCPUUsage | critical | Same as above | Identify and restart CPU hog if safe | +| HighSystemLoad | warning | `uptime`, `top -bn1`, `docker stats` | Report findings, identify load source | +| NFSMountDown | critical | `mount \| grep nfs`, `ping 192.168.86.30` | Attempt remount, report if NAS unreachable | +| NASDiskSpaceLow | warning | `df -h /mnt/docker-data` | Report, identify large consumers | +| SlowHTTPResponse | warning | `curl -w` timing, `docker logs` | Restart slow service | +| FrigateHighMemory | warning | `docker stats frigate` | `docker restart frigate` | +| ServiceHighMemory | warning | `docker stats`, identify container | Restart if safe | +| ServerDown | critical | SSH attempt, ping | Report — cannot fix hardware | +| ServerRebooted | info | Check uptime, verify services | Report, verify all containers are running | +| HighSwapUsage | warning | `free -m`, `docker stats` | Identify swap contributor, restart | +| CriticalSwapUsage | critical | Same as above | Restart heaviest swap user | +| PrometheusTargetDown | warning | `curl` the target, `docker ps` | Restart target service if down | +| Watchdog | info | N/A | Ignore (dead-man's switch, silenced in Alertmanager) | +| UnifiDeviceDown | warning | Check Unifi controller | Report — network device issue | + +### Resolved Alerts + +When Spirit sees a "resolved" alert, acknowledge briefly (no investigation needed). + +### Catch-All Rule + +For any unrecognized alert type not in the table above: investigate and report findings, but do not take remediation action. + +## What Does Not Change + +- Spirit's Matrix account, access token, homeserver config +- Spirit's gateway port (18795) and OpenClaw config structure +- Other agents (margaret, clippy, bob, travis) +- Alertbot continues posting alerts — Spirit is additive +- Prometheus, Alertmanager, alert rules — all unchanged + +## Implementation Steps + +1. Build `spirit-ops:latest` image on hf-pdocker-01 +2. Generate SSH keypair, collect host keys, deploy public key to nodes +3. Write workspace identity files (SOUL.md, IDENTITY.md, USER.md) to NAS +4. Update compose file (image, volume mounts, memory limit) +5. Redeploy Spirit container +6. Invite Spirit to #notifications room (after new container is running) +7. Verify: check SSH connectivity, docker socket access, Matrix room presence +8. Test: send a test alert through Alertmanager, confirm Spirit investigates and responds diff --git a/stacks/monitoring/prometheus.yml b/stacks/monitoring/prometheus.yml index ae11ac0..ddded41 100644 --- a/stacks/monitoring/prometheus.yml +++ b/stacks/monitoring/prometheus.yml @@ -69,6 +69,8 @@ scrape_configs: - http://192.168.86.192:3001 # Grafana - http://192.168.86.192:9095 # Prometheus - http://192.168.86.192:9093 # Alertmanager + - http://192.168.86.192:18815 # Travis (NemoClaw agent) + - http://192.168.86.192:18795 # Spirit (infra ops agent) relabel_configs: - source_labels: [__address__] target_label: __param_target