jeremy/hobbs-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
9943fd44105535e047d1f99fac8b003501cfa578
hobbs-iac/stacks/monitoring/prometheus.yml
T
jeremy 9943fd4410 infra: repurpose Spirit as autonomous infrastructure ops agent 
Spirit now monitors #notifications Matrix room for Prometheus alerts,
auto-investigates issues, takes corrective action (container restarts,
disk cleanup, etc.), and documents incidents in Wiki.js.

Changes:
- Custom spirit-ops Docker image (Docker CLI + SSH + diagnostic tools)
- Docker socket mount for local container management
- SSH keys for remote access to hf-pdocker-02 and bart
- Wiki.js API integration for incident documentation
- Prometheus probe for Spirit gateway health
- Safety guardrails: protected services, rate limiting, escalation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-19 10:36:20 -05:00

176 lines
6.0 KiB
YAML
Raw Blame History

global:
scrape_interval: 1m
evaluation_interval: 1m
alerting:
alertmanagers:
- static_configs:
- targets:
- 'alertmanager:9093'
rule_files:
- "alerts.yml"
scrape_configs:
# Prometheus self-monitoring
- job_name: 'prometheus'
scrape_interval: 1m
static_configs:
- targets: ['localhost:9090']
# Node Exporter (system metrics) - all 3 swarm nodes
- job_name: 'node'
static_configs:
- targets: ['192.168.86.192:9100']
labels:
hostname: 'hf-pdocker-01'
host_ip: '192.168.86.192'
- targets: ['192.168.86.167:9100']
labels:
hostname: 'bart'
host_ip: '192.168.86.167'
- targets: ['192.168.86.100:9100']
labels:
hostname: 'hf-pdocker-02'
host_ip: '192.168.86.100'
# Blackbox Exporter self-monitoring
- job_name: 'blackbox'
static_configs:
- targets: ['blackbox-exporter:9115']
# Matrix Alertmanager Receiver metrics (now on hf-pdocker-01)
- job_name: 'matrix-alertmanager-receiver'
scrape_interval: 1m
static_configs:
- targets: ['192.168.86.192:9094']
labels:
hostname: 'hf-pdocker-01'
# HTTP Health Checks - hf-pdocker-01 services
- job_name: 'http-pdocker01'
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets:
- http://192.168.86.192:5000 # Frigate NVR (standalone, host network)
- http://192.168.86.192:5199/health # Frigate Notify
- http://192.168.86.192:5200/health # clip-service (CLIP embeddings)
- http://192.168.86.192:3010 # Open WebUI
- http://192.168.86.192:8888/health # Farm Manager API
- http://192.168.86.192:8889/health # Farm Agent
- http://192.168.86.192:8123 # Home Assistant
- http://192.168.86.192:6052 # ESPHome
- http://192.168.86.192:8199/guacamole/ # Guacamole
- http://192.168.86.192:8008 # Matrix Synapse
- http://192.168.86.192:3000 # AdGuard Home
- http://192.168.86.192:8090/dashboard/ # Traefik Dashboard
- http://192.168.86.192:3001 # Grafana
- http://192.168.86.192:9095 # Prometheus
- http://192.168.86.192:9093 # Alertmanager
- http://192.168.86.192:18815 # Travis (NemoClaw agent)
- http://192.168.86.192:18795 # Spirit (infra ops agent)
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
# HTTP Health Checks - hf-pdocker-02 services
- job_name: 'http-pdocker02'
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets:
- http://192.168.86.100:3003 # Gitea
- http://192.168.86.100:3004 # Semaphore
- http://192.168.86.100:3002 # Wiki.js
- http://192.168.86.100:26226 # Vaultwarden
- http://192.168.86.100:5006 # Actual Budget
- http://192.168.86.100:8001/openapi.json # MCPO
- http://192.168.86.100:8889/health # Farm Agent
- http://192.168.86.100:8888 # SearXNG
- http://192.168.86.100:8000/healthcheck # Unstructured API
- http://192.168.86.100:9000 # Portainer
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
# HTTPS Health Checks - services with self-signed certs
- job_name: 'http-pdocker02-https'
metrics_path: /probe
params:
module: [http_2xx_insecure]
static_configs:
- targets:
- https://192.168.86.100:8443 # Unifi Controller (self-signed cert)
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
# HTTP Health Checks - bart services
- job_name: 'http-bart'
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets:
- http://192.168.86.167:8096 # Jellyfin
- http://192.168.86.167:8989 # Sonarr
- http://192.168.86.167:7878 # Radarr
- http://192.168.86.167:8082 # SABnzbd
- http://192.168.86.167:5076 # NZBHydra2
- http://192.168.86.167:5055 # Jellyseerr
- http://192.168.86.167:9392 # OpenVAS
- http://192.168.86.167:8889/health # Farm Agent
- targets:
- http://192.168.86.167:3005 # Onyx
labels:
name: onyx
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
# HTTPS Health Checks - External domains via Traefik
- job_name: 'http-external'
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets:
- https://mess.hobbs.farm # Matrix (via Traefik)
- https://solo.hobbs.farm/guacamole/ # Guacamole (via Traefik)
- https://yoda.hobbs.farm # Home Assistant
- https://vault.hobbs.farm # Vaultwarden
- https://git.hobbs.farm # Gitea
- https://open.hobbs.farm # Open WebUI
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
# Unifi Network Devices - uncomment when unifi-exporter is deployed
# - job_name: 'unifi-devices'
# scrape_interval: 30s
# static_configs:
# - targets: ['unifi-exporter:9101']
Reference in New Issue View Git Blame Copy Permalink