infra: full IaC state after hf-pdocker-01 rebuild (2026-03-08)
Replaces initial Nomad/Consul/Terraform skeleton with current Docker-based infrastructure across all three nodes (hf-pdocker-01, hf-pdocker-02, bart). Key changes: - CLAUDE.md: updated service distribution reflecting hf-pdocker-01 restored - docs/plans/2026-03-08-hf-pdocker-01-rebuild.md: new rebuild plan doc - compose/bart/: ai-agents, guacamole, matrix, media, openvas, smarthome, watchtower - compose/hf-pdocker-02/: traefik, monitoring, adguard, gitea, semaphore, wikijs, unifi, vaultwarden, actual-budget, mcpo, netclaw, trivy, watchtower, n8n - compose/hf-pdocker-01/: frigate, frigate-notify, farm-agent - stacks/: all Swarm stack definitions (monitoring, traefik, smarthome, matrix, guacamole, media, ai-agents, frigate, gitea, semaphore, wikijs, unifi, etc.) - .gitignore: exclude .env credential files and acme.json certs - .env.example: placeholder credential files for all services - .claude/agents/: gitea-backup and other infrastructure agent configs - scripts/: frigate media cleanup and NAS sync scripts Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ["alertname"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receiver: "matrix-alerts"
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
repeat_interval: 30m
|
||||
receiver: "matrix-alerts"
|
||||
- match:
|
||||
alertname: Watchdog
|
||||
receiver: "null"
|
||||
repeat_interval: 24h
|
||||
- match:
|
||||
severity: info
|
||||
repeat_interval: 12h
|
||||
receiver: "matrix-alerts"
|
||||
|
||||
receivers:
|
||||
- name: "null"
|
||||
- name: "matrix-alerts"
|
||||
webhook_configs:
|
||||
- url: "http://matrix-alertmanager-receiver:9094/alerts/notifications"
|
||||
http_config:
|
||||
basic_auth:
|
||||
username: "alertmanager"
|
||||
password: "alertmanager-webhook"
|
||||
send_resolved: true
|
||||
|
||||
inhibit_rules:
|
||||
- source_match:
|
||||
severity: "critical"
|
||||
target_match:
|
||||
severity: "warning"
|
||||
equal: ["alertname", "instance"]
|
||||
@@ -0,0 +1,244 @@
|
||||
groups:
|
||||
- name: infrastructure_devices
|
||||
interval: 30s
|
||||
rules:
|
||||
# Unifi Device Down
|
||||
- alert: UnifiDeviceDown
|
||||
expr: unifi_device_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
type: infrastructure
|
||||
annotations:
|
||||
summary: "Unifi device {{ $labels.name }} is offline"
|
||||
description: "Device {{ $labels.name }} ({{ $labels.model }}) at {{ $labels.ip }} has been offline for more than 2 minutes."
|
||||
|
||||
- name: server_health
|
||||
interval: 60s
|
||||
rules:
|
||||
# Server Down
|
||||
- alert: ServerDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "Server {{ $labels.hostname }} is DOWN"
|
||||
description: "{{ $labels.hostname }} ({{ $labels.host_ip }}) is not responding to health checks for 1 minute."
|
||||
|
||||
# High disk usage
|
||||
- alert: HighDiskUsage
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: Disk space critically low"
|
||||
description: "Root filesystem on {{ $labels.hostname }} has less than 10% free space remaining."
|
||||
|
||||
# Moderate disk usage warning
|
||||
- alert: DiskSpaceWarning
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 20
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: Disk space low"
|
||||
description: "Root filesystem on {{ $labels.hostname }} has less than 20% free space."
|
||||
|
||||
# High memory usage
|
||||
- alert: HighMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: High memory usage"
|
||||
description: "Memory usage on {{ $labels.hostname }} is above 90% for 5 minutes."
|
||||
|
||||
# Critical memory usage
|
||||
- alert: CriticalMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 95
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: Critical memory usage"
|
||||
description: "Memory usage on {{ $labels.hostname }} is above 95%. System may become unstable."
|
||||
|
||||
# High CPU usage (bart: 90% threshold due to Frigate/media workloads, others: 80%)
|
||||
- alert: HighCPUUsage
|
||||
expr: >
|
||||
(100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname!="bart"}[5m])) * 100) > 80)
|
||||
or
|
||||
(100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname="bart"}[5m])) * 100) > 90)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: High CPU usage"
|
||||
description: "CPU usage on {{ $labels.hostname }} is above threshold for 10 minutes."
|
||||
|
||||
# Critical CPU usage
|
||||
- alert: CriticalCPUUsage
|
||||
expr: 100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: Critical CPU usage"
|
||||
description: "CPU usage on {{ $labels.hostname }} is above 95% for 5 minutes. Performance degraded."
|
||||
|
||||
# High system load
|
||||
- alert: HighSystemLoad
|
||||
expr: node_load5 / on(hostname) count by(hostname) (node_cpu_seconds_total{mode="idle"}) > 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: High system load"
|
||||
description: "5-minute load average on {{ $labels.hostname }} is more than 2x CPU count."
|
||||
|
||||
# Server rebooted
|
||||
- alert: ServerRebooted
|
||||
expr: node_time_seconds - node_boot_time_seconds < 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }} rebooted"
|
||||
description: "{{ $labels.hostname }} was rebooted less than 10 minutes ago."
|
||||
|
||||
# High swap usage
|
||||
- alert: HighSwapUsage
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 50)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: High swap usage"
|
||||
description: "Swap usage on {{ $labels.hostname }} is above 50% for 5 minutes. Check container memory limits."
|
||||
|
||||
# Critical swap usage
|
||||
- alert: CriticalSwapUsage
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 80)
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: Critical swap usage"
|
||||
description: "Swap usage on {{ $labels.hostname }} is above 80%. System may become unresponsive."
|
||||
|
||||
# NFS mount failure (all data is on NAS - this is critical)
|
||||
- alert: NFSMountDown
|
||||
expr: node_filesystem_avail_bytes{fstype="nfs"} == 0 or absent(node_filesystem_avail_bytes{fstype="nfs"})
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
type: server
|
||||
annotations:
|
||||
summary: "{{ $labels.hostname }}: NFS mount unavailable"
|
||||
description: "NFS mount on {{ $labels.hostname }} at {{ $labels.mountpoint }} is down. All services depend on NAS storage."
|
||||
|
||||
# NAS disk space low
|
||||
- alert: NASDiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{fstype="nfs",mountpoint="/mnt/docker-data"} / node_filesystem_size_bytes{fstype="nfs",mountpoint="/mnt/docker-data"}) * 100 < 15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
type: server
|
||||
annotations:
|
||||
summary: "NAS disk space low"
|
||||
description: "NFS share /mnt/docker-data has less than 15% free space."
|
||||
|
||||
- name: docker_containers
|
||||
interval: 60s
|
||||
rules:
|
||||
# Service/container unreachable via blackbox HTTP probe
|
||||
# Matches all http-pdocker01, http-pdocker02, http-bart, and http-external jobs
|
||||
- alert: ContainerDown
|
||||
expr: probe_success{job=~"http-.*"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
type: container
|
||||
annotations:
|
||||
summary: "Service unreachable: {{ $labels.instance }}"
|
||||
description: "Service at {{ $labels.instance }} is not responding to HTTP health checks for 3 minutes."
|
||||
|
||||
- name: service_health
|
||||
interval: 60s
|
||||
rules:
|
||||
# Service responding but very slow
|
||||
- alert: SlowHTTPResponse
|
||||
expr: probe_duration_seconds{job=~"http-.*"} > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: container
|
||||
annotations:
|
||||
summary: "Slow response: {{ $labels.instance }}"
|
||||
description: "{{ $labels.instance }} is responding but taking more than 10 seconds."
|
||||
|
||||
# Prometheus scrape target down
|
||||
- alert: PrometheusTargetDown
|
||||
expr: up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus target down: {{ $labels.job }}"
|
||||
description: "Prometheus cannot scrape {{ $labels.instance }} (job: {{ $labels.job }}) for 5 minutes."
|
||||
|
||||
# Alerting pipeline dead-man's switch (Watchdog always fires - if it stops, pipeline is broken)
|
||||
- alert: Watchdog
|
||||
expr: vector(1)
|
||||
labels:
|
||||
severity: none
|
||||
type: monitoring
|
||||
annotations:
|
||||
summary: "Alerting pipeline active"
|
||||
description: "This is a dead-man's-switch alert. If you stop receiving this, the alerting pipeline is broken."
|
||||
|
||||
- name: frigate_alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
# Frigate high memory (standalone container on hf-pdocker-01 with host network)
|
||||
- alert: FrigateHighMemory
|
||||
expr: container_memory_usage_bytes{name="frigate"} / container_spec_memory_limit_bytes{name="frigate"} > 0.85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: frigate
|
||||
annotations:
|
||||
summary: "Frigate high memory usage"
|
||||
description: "Frigate is using more than 85% of allocated memory."
|
||||
|
||||
- name: service_resources
|
||||
interval: 60s
|
||||
rules:
|
||||
# Any service container using more than 90% of its memory limit
|
||||
- alert: ServiceHighMemory
|
||||
expr: >
|
||||
(container_memory_usage_bytes{name!="",name!~".*_db.*",name!~".*db$"} /
|
||||
container_spec_memory_limit_bytes{name!="",name!~".*_db.*",name!~".*db$"} > 0)
|
||||
> 0.90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
type: container
|
||||
annotations:
|
||||
summary: "{{ $labels.name }}: High memory usage"
|
||||
description: "Container {{ $labels.name }} on {{ $labels.instance }} is using more than 90% of its memory limit."
|
||||
@@ -0,0 +1,17 @@
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: "ip4"
|
||||
http_2xx_insecure:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: "ip4"
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
@@ -0,0 +1,197 @@
|
||||
services:
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
restart: unless-stopped
|
||||
user: "104"
|
||||
ports:
|
||||
- "3001:3000"
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/grafana:/var/lib/grafana
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/api/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9095:9090"
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
- '--web.enable-lifecycle'
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/prometheus:/etc/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:9090/-/healthy || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9093:9093"
|
||||
command:
|
||||
- '--config.file=/etc/alertmanager/alertmanager.yml'
|
||||
- '--storage.path=/alertmanager'
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/alertmanager:/etc/alertmanager
|
||||
- alertmanager-data:/alertmanager
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:9093/-/healthy || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9100:9100"
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--path.sysfs=/host/sys'
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: global
|
||||
resources:
|
||||
limits:
|
||||
memory: 128M
|
||||
|
||||
influxdb:
|
||||
image: influxdb:1.8-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/influxdb:/var/lib/influxdb
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:8086/ping || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
|
||||
matrix-alertmanager-receiver:
|
||||
image: metio/matrix-alertmanager-receiver:2026.2.25
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9094:9094"
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/alertmanager/matrix-receiver-config.yml:/config/config.yml:ro
|
||||
command: ["--config-path", "/config/config.yml"]
|
||||
# NOTE: No healthcheck - this image is distroless (no shell), so CMD-SHELL healthchecks fail.
|
||||
# The service is verified healthy by Alertmanager successfully POSTing to it.
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 128M
|
||||
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9115:9115"
|
||||
command:
|
||||
- '--config.file=/config/blackbox.yml'
|
||||
volumes:
|
||||
- /mnt/docker-data/configs/blackbox:/config:ro
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == hf-pdocker-01
|
||||
resources:
|
||||
limits:
|
||||
memory: 128M
|
||||
|
||||
volumes:
|
||||
alertmanager-data:
|
||||
@@ -0,0 +1,55 @@
|
||||
http:
|
||||
address: 0.0.0.0
|
||||
port: 9094
|
||||
alerts-path-prefix: /alerts
|
||||
metrics-path: /metrics
|
||||
metrics-enabled: true
|
||||
# basic-password is required - the app has a hardcoded default basic-username of "alertmanager"
|
||||
# and will exit silently if basic-password is not set
|
||||
basic-password: "alertmanager-webhook"
|
||||
|
||||
matrix:
|
||||
# Matrix Synapse is on bart (failover from hf-pdocker-01)
|
||||
homeserver-url: http://192.168.86.167:8008
|
||||
user-id: "@alertbot:hobbs.farm"
|
||||
# Access token stored directly - ${MATRIX_ACCESS_TOKEN} env var syntax is NOT supported
|
||||
# (the app uses yaml parsing, not shell interpolation)
|
||||
access-token: "syt_YWxlcnRib3Q_eOVAYjYKWBZxymfdxhPE_0iYlOZ"
|
||||
room-mapping:
|
||||
notifications: "!jYmqxbzEsvRgKYvwMP:hobbs.farm"
|
||||
|
||||
templating:
|
||||
external-url-mapping:
|
||||
# Alertmanager and Prometheus are on hf-pdocker-02 (failover from hf-pdocker-01)
|
||||
"http://alertmanager:9093": "http://192.168.86.100:9093"
|
||||
generator-url-mapping:
|
||||
"http://prometheus:9090": "http://192.168.86.100:9095"
|
||||
computed-values:
|
||||
- values:
|
||||
color: red
|
||||
when-matching-labels:
|
||||
severity: critical
|
||||
- values:
|
||||
color: orange
|
||||
when-matching-labels:
|
||||
severity: warning
|
||||
- values:
|
||||
color: blue
|
||||
when-matching-labels:
|
||||
severity: info
|
||||
# NOTE: Templates use .Alert (singular) not .Alerts (plural)
|
||||
# This application calls the template once per individual alert, not per group
|
||||
firing-template: |
|
||||
<h3>ALERT: {{ .Alert.Labels.alertname }}</h3>
|
||||
<table>
|
||||
<tr><td><b>Instance:</b></td><td>{{ .Alert.Labels.instance }}</td></tr>
|
||||
<tr><td><b>Severity:</b></td><td>{{ .Alert.Labels.severity }}</td></tr>
|
||||
<tr><td><b>Summary:</b></td><td>{{ .Alert.Annotations.summary }}</td></tr>
|
||||
<tr><td><b>Description:</b></td><td>{{ .Alert.Annotations.description }}</td></tr>
|
||||
</table>
|
||||
resolved-template: |
|
||||
<h3>RESOLVED: {{ .Alert.Labels.alertname }}</h3>
|
||||
<table>
|
||||
<tr><td><b>Instance:</b></td><td>{{ .Alert.Labels.instance }}</td></tr>
|
||||
<tr><td><b>Summary:</b></td><td>{{ .Alert.Annotations.summary }}</td></tr>
|
||||
</table>
|
||||
@@ -0,0 +1,163 @@
|
||||
global:
|
||||
scrape_interval: 1m
|
||||
evaluation_interval: 1m
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- 'alertmanager:9093'
|
||||
|
||||
rule_files:
|
||||
- "alerts.yml"
|
||||
|
||||
scrape_configs:
|
||||
# Prometheus self-monitoring
|
||||
- job_name: 'prometheus'
|
||||
scrape_interval: 1m
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Node Exporter (system metrics) - all 3 swarm nodes
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets: ['192.168.86.192:9100']
|
||||
labels:
|
||||
hostname: 'hf-pdocker-01'
|
||||
host_ip: '192.168.86.192'
|
||||
- targets: ['192.168.86.167:9100']
|
||||
labels:
|
||||
hostname: 'bart'
|
||||
host_ip: '192.168.86.167'
|
||||
- targets: ['192.168.86.100:9100']
|
||||
labels:
|
||||
hostname: 'hf-pdocker-02'
|
||||
host_ip: '192.168.86.100'
|
||||
|
||||
# Blackbox Exporter self-monitoring
|
||||
- job_name: 'blackbox'
|
||||
static_configs:
|
||||
- targets: ['blackbox-exporter:9115']
|
||||
|
||||
# Matrix Alertmanager Receiver metrics (running on hf-pdocker-02 during failover)
|
||||
- job_name: 'matrix-alertmanager-receiver'
|
||||
scrape_interval: 1m
|
||||
static_configs:
|
||||
- targets: ['192.168.86.100:9094']
|
||||
labels:
|
||||
hostname: 'hf-pdocker-02'
|
||||
|
||||
# HTTP Health Checks - hf-pdocker-01 services (native services only; failover services tracked in other jobs)
|
||||
- job_name: 'http-pdocker01'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://192.168.86.192:5000 # Frigate NVR (standalone, host network)
|
||||
- http://192.168.86.192:5199/health # Frigate Notify
|
||||
- http://192.168.86.192:8888/health # Farm Manager API
|
||||
- http://192.168.86.192:8889/health # Farm Agent
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# HTTP Health Checks - hf-pdocker-02 services (native + failover from hf-pdocker-01)
|
||||
- job_name: 'http-pdocker02'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://192.168.86.100:3003 # Gitea
|
||||
- http://192.168.86.100:3004 # Semaphore
|
||||
- http://192.168.86.100:3002 # Wiki.js
|
||||
- http://192.168.86.100:26226 # Vaultwarden
|
||||
- http://192.168.86.100:5006 # Actual Budget
|
||||
- http://192.168.86.100:8001/openapi.json # MCPO
|
||||
- http://192.168.86.100:8889/health # Farm Agent
|
||||
- http://192.168.86.100:3001 # Grafana (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.100:9095 # Prometheus (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.100:9093 # Alertmanager (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.100:3000 # AdGuard Home (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.100:9000 # Portainer
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# HTTPS Health Checks - services with self-signed certs
|
||||
- job_name: 'http-pdocker02-https'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx_insecure]
|
||||
static_configs:
|
||||
- targets:
|
||||
- https://192.168.86.100:8443 # Unifi Controller (self-signed cert)
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# HTTP Health Checks - bart services (native + failover from hf-pdocker-01)
|
||||
- job_name: 'http-bart'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://192.168.86.167:8096 # Jellyfin
|
||||
- http://192.168.86.167:8989 # Sonarr
|
||||
- http://192.168.86.167:7878 # Radarr
|
||||
- http://192.168.86.167:8082 # SABnzbd
|
||||
- http://192.168.86.167:5076 # NZBHydra2
|
||||
- http://192.168.86.167:5055 # Jellyseerr
|
||||
- http://192.168.86.167:9392 # OpenVAS
|
||||
- http://192.168.86.167:8889/health # Farm Agent
|
||||
- http://192.168.86.167:8123 # Home Assistant (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.167:6052 # ESPHome (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.167:8199/guacamole/ # Guacamole (FAILOVER from hf-pdocker-01)
|
||||
- http://192.168.86.167:8008 # Matrix Synapse (FAILOVER from hf-pdocker-01)
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# HTTPS Health Checks - External domains via Traefik
|
||||
- job_name: 'http-external'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- https://mess.hobbs.farm # Matrix (via Traefik)
|
||||
- https://solo.hobbs.farm/guacamole/ # Guacamole (via Traefik)
|
||||
- https://yoda.hobbs.farm # Home Assistant
|
||||
- https://vault.hobbs.farm # Vaultwarden
|
||||
- https://git.hobbs.farm # Gitea
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# Unifi Network Devices - uncomment when unifi-exporter is deployed
|
||||
# - job_name: 'unifi-devices'
|
||||
# scrape_interval: 30s
|
||||
# static_configs:
|
||||
# - targets: ['unifi-exporter:9101']
|
||||
Reference in New Issue
Block a user