jeremy/hobbs-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: recover from 2026-07-12 power-event fallout + efficiency cleanup

Browse Source 
- gitea: LevelDB queues -> local /var/lib/gitea-queues (stale NFS lock, 7,972-restart loop)
- semaphore-db: data -> local /var/lib/semaphore-db (stale ibdata1 NFS lock, 975 restarts)
- sonarr: config -> local /var/lib/sonarr (SQLite-on-NFS migration crash)
- nzbhydra2: mem_limit 768M -> 1G (was pinned at limit, unhealthy)
- jellyfin: deploy pending VAAPI device passthrough (i965/renderD128)
- openvas: disable Watchtower (latest major pull forced pg_upgrade crash-loop)
- onyx: fully decommissioned (down since ~June 8 unnoticed)
- remove duplicate monitoring/traefik/guacamole stacks from .100, failover guacamole from bart
- synapse: raise rc_message (alertmanager receiver was 429-limited; alerts silently dropped for weeks)
- prometheus: UniFi probe 8443 -> 11443, solo.hobbs.farm via 401-tolerant module, drop onyx target
- frigate restart cron: unwritable /var/log path meant it never ran; now 4x daily to ~glyptic
- CLAUDE.md: document all of the above + AdGuard broken/unconfigured since 2026-05-17

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jeremy
2026-07-13 19:20:10 -05:00
parent 81a7464dd8
commit 5d7c8ea030
7 changed files with 37 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/bart/media/docker-compose.yml
+10 -2
View File
@@ -6,11 +6,17 @@ services:
environment:
- PUID=1000
- PGID=1000
- LIBVA_DRIVER_NAME=i965 # Haswell HD 4600 uses legacy i965 (iHD is Broadwell+)
volumes:
- /mnt/docker-data/configs/jellyfin:/config
- /mnt/nas-media:/media
ports:
- "8096:8096"
devices:
- /dev/dri:/dev/dri # Intel iGPU for VAAPI hardware transcoding
group_add:
- "993" # render
- "44" # video
mem_limit: 2G
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:8096/health || exit 1"]
@@ -33,7 +39,9 @@ services:
- PGID=1000
- TZ=America/Chicago
volumes:
- /mnt/docker-data/configs/sonarr:/config
# LOCAL disk, not NFS: SQLite-on-NFS locking crashed Sonarr's DB migration
# after the 2026-07-12 power event ("waiting for user intervention")
- /var/lib/sonarr:/config
- /mnt/nas-media:/media
ports:
- "8989:8989"
@@ -114,7 +122,7 @@ services:
- /mnt/docker-data/configs/nzbhydra:/config
ports:
- "5076:5076"
mem_limit: 768M
mem_limit: 1G
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:5076/ || exit 1"]
interval: 30s
compose/bart/onyx/docker-compose.yml
-231
View File
@@ -1,231 +0,0 @@
services:
api_server:
image: onyxdotapp/onyx-backend:latest
command: >
/bin/sh -c "alembic upgrade head &&
echo Starting Onyx API Server &&
uvicorn onyx.main:app --host 0.0.0.0 --port 8080"
depends_on:
relational_db:
condition: service_healthy
index:
condition: service_healthy
cache:
condition: service_healthy
restart: "no"
mem_limit: 2G
memswap_limit: 2G
env_file:
- .env
environment:
- AUTH_TYPE=basic
- DISABLE_TELEMETRY=true
- POSTGRES_HOST=relational_db
- VESPA_HOST=index
- REDIS_HOST=cache
- MODEL_SERVER_HOST=inference_model_server
- INDEXING_MODEL_SERVER_HOST=indexing_model_server
- OPENSEARCH_HOST=opensearch
- ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=false
- FILE_STORE_BACKEND=postgres
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- onyx_file_connector:/home/file_connector_storage
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
background:
image: onyxdotapp/onyx-backend:latest
command: >
/bin/sh -c "/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf"
depends_on:
relational_db:
condition: service_healthy
index:
condition: service_healthy
cache:
condition: service_healthy
restart: "no"
mem_limit: 3G
memswap_limit: 3G
env_file:
- .env
environment:
- AUTH_TYPE=basic
- DISABLE_TELEMETRY=true
- POSTGRES_HOST=relational_db
- VESPA_HOST=index
- REDIS_HOST=cache
- MODEL_SERVER_HOST=inference_model_server
- INDEXING_MODEL_SERVER_HOST=indexing_model_server
- OPENSEARCH_HOST=opensearch
- ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=false
- FILE_STORE_BACKEND=postgres
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- onyx_file_connector:/home/file_connector_storage
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
web_server:
image: onyxdotapp/onyx-web-server:latest
depends_on:
- api_server
restart: "no"
mem_limit: 512M
memswap_limit: 512M
environment:
- INTERNAL_URL=http://api_server:8080
- WEB_DOMAIN=https://onyx.hobbs.farm
- NEXT_PUBLIC_DISABLE_STREAMING=false
- AUTH_TYPE=basic
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
inference_model_server:
image: onyxdotapp/onyx-model-server:latest
command: >
/bin/sh -c "exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000"
restart: "no"
mem_limit: 2G
memswap_limit: 2G
environment:
- MIN_THREADS_ML_MODELS=1
- LOG_LEVEL=info
volumes:
- onyx_model_cache_huggingface:/root/.cache/huggingface/
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
indexing_model_server:
image: onyxdotapp/onyx-model-server:latest
command: >
/bin/sh -c "exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000"
restart: "no"
mem_limit: 2G
memswap_limit: 2G
environment:
- MIN_THREADS_ML_MODELS=1
- INDEXING_ONLY=true
- LOG_LEVEL=info
volumes:
- onyx_model_cache_huggingface:/root/.cache/huggingface/
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
relational_db:
image: postgres:15.2-alpine
restart: "no"
mem_limit: 512M
memswap_limit: 512M
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=onyx_postgres_pw
volumes:
- onyx_db:/var/lib/postgresql/data
networks:
- onyx
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
labels:
com.centurylinklabs.watchtower.enable: "false"
index:
image: vespaengine/vespa:8.609.39
restart: "no"
mem_limit: 4G
memswap_limit: 4G
volumes:
- onyx_vespa:/opt/vespa/var
networks:
- onyx
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:19071/state/v1/health || exit 1"]
interval: 30s
timeout: 10s
retries: 10
start_period: 120s
labels:
com.centurylinklabs.watchtower.enable: "false"
opensearch:
image: opensearchproject/opensearch:2.11.1
restart: "no"
mem_limit: 2G
memswap_limit: 2G
environment:
- discovery.type=single-node
- plugins.security.disabled=true
- OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
- DISABLE_INSTALL_DEMO_CONFIG=true
volumes:
- onyx_opensearch:/usr/share/opensearch/data
networks:
- onyx
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
interval: 30s
timeout: 10s
retries: 10
start_period: 120s
labels:
com.centurylinklabs.watchtower.enable: "false"
cache:
image: redis:7.4-alpine
restart: "no"
mem_limit: 256M
memswap_limit: 256M
command: redis-server --maxmemory 128mb --maxmemory-policy allkeys-lru
networks:
- onyx
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 5
labels:
com.centurylinklabs.watchtower.enable: "false"
nginx:
image: nginx:1.25.5-alpine
restart: "no"
mem_limit: 128M
memswap_limit: 128M
depends_on:
- api_server
- web_server
ports:
- "3005:80"
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
networks:
- onyx
labels:
com.centurylinklabs.watchtower.enable: "false"
volumes:
onyx_file_connector:
onyx_model_cache_huggingface:
onyx_db:
onyx_vespa:
onyx_opensearch:
networks:
onyx:
driver: bridge
compose/bart/onyx/nginx.conf
-36
View File
@@ -1,36 +0,0 @@
server {
listen 80;
server_name _;
# Use Docker's internal DNS with a short TTL so container IP changes
# (e.g. after docker compose up -d for a single service) are picked up
# without requiring an nginx reload.
resolver 127.0.0.11 valid=10s;
client_max_body_size 100M;
location /api/ {
set $api_upstream http://api_server:8080;
rewrite ^/api/(.*)$ /$1 break;
proxy_pass $api_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 300s;
proxy_connect_timeout 75s;
proxy_buffering off;
proxy_cache off;
}
location / {
set $web_upstream http://web_server:3000;
proxy_pass $web_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 300s;
proxy_connect_timeout 75s;
}
}
compose/bart/openvas/docker-compose.yml
+4
View File
@@ -3,6 +3,10 @@ services:
image: immauss/openvas:latest
container_name: openvas
restart: unless-stopped
labels:
# Watchtower pulled a new major that forced a pg_upgrade crash-loop
# (212 restarts, 2026-07-13); update OpenVAS deliberately instead
- "com.centurylinklabs.watchtower.enable=false"
environment:
- PASSWORD=${OPENVAS_PASSWORD}
- USERNAME=${OPENVAS_USERNAME}
compose/hf-pdocker-02/gitea/docker-compose.yml
+3
View File
@@ -19,6 +19,9 @@ services:
- GITEA__server__SSH_PORT=2222
volumes:
- /mnt/docker-data/configs/gitea:/data
# LOCAL disk for LevelDB queues: stale NFS lock after 2026-07-12 power event
# caused a 28h crash loop (7,972 restarts)
- /var/lib/gitea-queues:/data/gitea/queues
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
depends_on:
compose/hf-pdocker-02/semaphore/docker-compose.yml
+3 -1
View File
@@ -53,7 +53,9 @@ services:
- MYSQL_USER=semaphore
- MYSQL_PASSWORD=${SEMAPHORE_DB_PASSWORD}
volumes:
- /mnt/docker-data/databases/semaphore-db:/var/lib/mysql
# LOCAL disk, not NFS: MySQL on NFS suffers stale-lock and InnoDB corruption
# (2026-07-11 corruption, 2026-07-13 stale ibdata1 lock after power event)
- /var/lib/semaphore-db:/var/lib/mysql
networks:
- semaphore-internal
labels: