From 6f485ff6e5cd45aa5ec8ee2db2bf7881a0e0cc97 Mon Sep 17 00:00:00 2001 From: jeremy Date: Wed, 20 May 2026 08:06:05 -0500 Subject: [PATCH] =?UTF-8?q?docs:=20update=20CLAUDE.md=20=E2=80=94=20Coral?= =?UTF-8?q?=20USB=20TPU,=20Onyx=20redeploy,=20infra=20fixes=202026-05-20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 10 ++++++++-- stacks/monitoring/alerts.yml | 13 +++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b771093..961e1de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ This repository manages a **homelab infrastructure** with self-hosted services r ``` hf-pdocker-01 (23GB RAM, i7-7700T, NVMe, HD 630 GPU) -- RESTORED (2026-03-08, hardware repaired) -├── [COMPOSE] Frigate NVR (:5000) -- OpenVINO GPU detector, 13 cameras active +├── [COMPOSE] Frigate NVR (:5000) -- Coral USB TPU (edgetpu), ~11ms inference, 13 cameras active ├── [COMPOSE] Frigate-Notify v2 (:5199) -- MQTT->AI->Matrix notifications ├── [COMPOSE] clip-service (:5200) -- CLIP embeddings (ViT-L-14, CPU) ├── [COMPOSE] Home Assistant (:8123) @@ -44,6 +44,7 @@ hf-pdocker-01 (23GB RAM, i7-7700T, NVMe, HD 630 GPU) -- RESTORED (2026-03-08, ha bart (15GB RAM, i5-4570T, HDD) ├── [COMPOSE] Media Stack (Jellyfin, Sonarr, Radarr, SABnzbd, NZBHydra2, Jellyseerr) +├── [COMPOSE] Onyx (:3005) -- redeployed 2026-05-20, all services :latest, Watchtower disabled on all 10 ├── [COMPOSE] OpenVAS (vulnerability scanner) ├── [COMPOSE] Farm Agent (:8889) ├── [COMPOSE] Node-exporter (:9100) -- standalone with pid:host @@ -88,7 +89,8 @@ All service data lives on the **Synology NAS** via NFS, mounted at `/mnt/docker- └── bart/{media,openvas,watchtower}/ ``` -**NFS mount (all nodes):** `192.168.86.30:/volume1/Docker /mnt/docker-data nfs rw,soft,timeo=300,intr,_netdev,nfsvers=3 0 0` +**NFS mount (all nodes):** `192.168.86.30:/volume1/Docker /mnt/docker-data nfs rw,soft,timeo=300,intr,_netdev,nfsvers=3,nofail,x-systemd.after=network-online.target,x-systemd.mount-timeout=60s 0 0` +**Note (hf-pdocker-01):** Both NFS mounts hardened with `nofail,x-systemd.after=network-online.target,x-systemd.mount-timeout=60s` on 2026-05-20 to prevent boot-time race (network not up when NFS tried to mount, cascading ~10 container failures). **NFS + Databases:** Database containers on NFS require `user: "1024:100"` to avoid `chown` failures from NFS root_squash. UID 1024 is the NAS anonymous user. @@ -291,3 +293,7 @@ All persistent data on NAS at `/mnt/docker-data/`: 5. **Frigate cameras**: Cameras on 192.168.44.x subnet may show RTSP timeouts (check camera network) 6. **Registry container**: Stopped on hf-pdocker-01 to free port 5000 for Frigate 7. **systemd-resolved DNSStubListener**: Must be disabled on any node running AdGuard Home (port 53 conflict). Fix: `/etc/systemd/resolved.conf.d/nostub.conf` with `DNSStubListener=no`, then `systemctl restart systemd-resolved`. Applied on hf-pdocker-01 and hf-pdocker-02. +8. **NFS boot-mount race (hf-pdocker-01)**: NFS mounts failed at boot (network not ready in time). Fixed 2026-05-20 by adding `nofail,x-systemd.after=network-online.target,x-systemd.mount-timeout=60s` to both fstab entries. If mounts are missing after reboot: `sudo mount /mnt/docker-data && sudo mount /mnt/infrastructure`. +9. **Guacamole schema**: App image pinned to `guacamole/guacamole:1.6.0` + `guacd:1.6.0` with Watchtower disabled. DB schema was reset to fresh 1.6.0 on 2026-05-20 (old schema was non-standard). Recreate connections after any DB reset. +10. **Gitea image**: Pinned to `gitea/gitea:1.26.1` with Watchtower disabled (2026-05-20). Watchtower previously pulled broken `:latest` images causing missing git binary errors. +11. **Home Assistant reverse proxy**: Config at `/mnt/docker-data/configs/homeassistant/configuration.yaml` (no hyphen). Must have `http: use_x_forwarded_for: true` + `trusted_proxies: [172.16.0.0/12, 127.0.0.1]` or proxied requests return 400. diff --git a/stacks/monitoring/alerts.yml b/stacks/monitoring/alerts.yml index 383f390..68aaa5e 100644 --- a/stacks/monitoring/alerts.yml +++ b/stacks/monitoring/alerts.yml @@ -118,27 +118,28 @@ groups: summary: "{{ $labels.hostname }} rebooted" description: "{{ $labels.hostname }} was rebooted less than 10 minutes ago." - # High swap usage + # High swap usage - uses absolute free swap threshold to avoid node-exporter v1.10.x + # SwapTotal double-counting bug on swapfile systems. Triggers when free swap < 1.5GB. - alert: HighSwapUsage - expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 50) + expr: (node_memory_SwapTotal_bytes > 0) and (node_memory_SwapFree_bytes < 1.5 * 1024 * 1024 * 1024) for: 5m labels: severity: warning type: server annotations: summary: "{{ $labels.hostname }}: High swap usage" - description: "Swap usage on {{ $labels.hostname }} is above 50% for 5 minutes. Check container memory limits." + description: "Swap usage on {{ $labels.hostname }} is high (free swap below 1.5GB for 5 minutes). Check container memory limits." - # Critical swap usage + # Critical swap usage - triggers when free swap < 512MB - alert: CriticalSwapUsage - expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 80) + expr: (node_memory_SwapTotal_bytes > 0) and (node_memory_SwapFree_bytes < 512 * 1024 * 1024) for: 2m labels: severity: critical type: server annotations: summary: "{{ $labels.hostname }}: Critical swap usage" - description: "Swap usage on {{ $labels.hostname }} is above 80%. System may become unresponsive." + description: "Swap usage on {{ $labels.hostname }} is critical (free swap below 512MB). System may become unresponsive." # NFS mount failure (all data is on NAS - this is critical) - alert: NFSMountDown