246 lines
9.3 KiB
YAML
246 lines
9.3 KiB
YAML
groups:
|
|
- name: infrastructure_devices
|
|
interval: 30s
|
|
rules:
|
|
# Unifi Device Down
|
|
- alert: UnifiDeviceDown
|
|
expr: unifi_device_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
type: infrastructure
|
|
annotations:
|
|
summary: "Unifi device {{ $labels.name }} is offline"
|
|
description: "Device {{ $labels.name }} ({{ $labels.model }}) at {{ $labels.ip }} has been offline for more than 2 minutes."
|
|
|
|
- name: server_health
|
|
interval: 60s
|
|
rules:
|
|
# Server Down
|
|
- alert: ServerDown
|
|
expr: up{job="node"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "Server {{ $labels.hostname }} is DOWN"
|
|
description: "{{ $labels.hostname }} ({{ $labels.host_ip }}) is not responding to health checks for 1 minute."
|
|
|
|
# High disk usage
|
|
- alert: HighDiskUsage
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: Disk space critically low"
|
|
description: "Root filesystem on {{ $labels.hostname }} has less than 10% free space remaining."
|
|
|
|
# Moderate disk usage warning
|
|
- alert: DiskSpaceWarning
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 20
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: Disk space low"
|
|
description: "Root filesystem on {{ $labels.hostname }} has less than 20% free space."
|
|
|
|
# High memory usage
|
|
- alert: HighMemoryUsage
|
|
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: High memory usage"
|
|
description: "Memory usage on {{ $labels.hostname }} is above 90% for 5 minutes."
|
|
|
|
# Critical memory usage
|
|
- alert: CriticalMemoryUsage
|
|
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 95
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: Critical memory usage"
|
|
description: "Memory usage on {{ $labels.hostname }} is above 95%. System may become unstable."
|
|
|
|
# High CPU usage (bart: 90% threshold due to Frigate/media workloads, others: 80%)
|
|
- alert: HighCPUUsage
|
|
expr: >
|
|
(100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname!="bart"}[5m])) * 100) > 80)
|
|
or
|
|
(100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname="bart"}[5m])) * 100) > 90)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: High CPU usage"
|
|
description: "CPU usage on {{ $labels.hostname }} is above threshold for 10 minutes."
|
|
|
|
# Critical CPU usage
|
|
- alert: CriticalCPUUsage
|
|
expr: 100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: Critical CPU usage"
|
|
description: "CPU usage on {{ $labels.hostname }} is above 95% for 5 minutes. Performance degraded."
|
|
|
|
# High system load
|
|
- alert: HighSystemLoad
|
|
expr: node_load5 / on(hostname) count by(hostname) (node_cpu_seconds_total{mode="idle"}) > 2
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: High system load"
|
|
description: "5-minute load average on {{ $labels.hostname }} is more than 2x CPU count."
|
|
|
|
# Server rebooted
|
|
- alert: ServerRebooted
|
|
expr: node_time_seconds - node_boot_time_seconds < 600
|
|
for: 1m
|
|
labels:
|
|
severity: info
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }} rebooted"
|
|
description: "{{ $labels.hostname }} was rebooted less than 10 minutes ago."
|
|
|
|
# High swap usage - uses absolute free swap threshold to avoid node-exporter v1.10.x
|
|
# SwapTotal double-counting bug on swapfile systems. Triggers when free swap < 1.5GB.
|
|
- alert: HighSwapUsage
|
|
expr: (node_memory_SwapTotal_bytes > 0) and (node_memory_SwapFree_bytes < 1.5 * 1024 * 1024 * 1024)
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: High swap usage"
|
|
description: "Swap usage on {{ $labels.hostname }} is high (free swap below 1.5GB for 5 minutes). Check container memory limits."
|
|
|
|
# Critical swap usage - triggers when free swap < 512MB
|
|
- alert: CriticalSwapUsage
|
|
expr: (node_memory_SwapTotal_bytes > 0) and (node_memory_SwapFree_bytes < 512 * 1024 * 1024)
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: Critical swap usage"
|
|
description: "Swap usage on {{ $labels.hostname }} is critical (free swap below 512MB). System may become unresponsive."
|
|
|
|
# NFS mount failure (all data is on NAS - this is critical)
|
|
- alert: NFSMountDown
|
|
expr: node_filesystem_avail_bytes{fstype="nfs"} == 0 or absent(node_filesystem_avail_bytes{fstype="nfs"})
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
type: server
|
|
annotations:
|
|
summary: "{{ $labels.hostname }}: NFS mount unavailable"
|
|
description: "NFS mount on {{ $labels.hostname }} at {{ $labels.mountpoint }} is down. All services depend on NAS storage."
|
|
|
|
# NAS disk space low
|
|
- alert: NASDiskSpaceLow
|
|
expr: (node_filesystem_avail_bytes{fstype="nfs",mountpoint="/mnt/docker-data"} / node_filesystem_size_bytes{fstype="nfs",mountpoint="/mnt/docker-data"}) * 100 < 15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
type: server
|
|
annotations:
|
|
summary: "NAS disk space low"
|
|
description: "NFS share /mnt/docker-data has less than 15% free space."
|
|
|
|
- name: docker_containers
|
|
interval: 60s
|
|
rules:
|
|
# Service/container unreachable via blackbox HTTP probe
|
|
# Matches all http-pdocker01, http-pdocker02, http-bart, and http-external jobs
|
|
- alert: ContainerDown
|
|
expr: probe_success{job=~"http-.*"} == 0
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
type: container
|
|
annotations:
|
|
summary: "Service unreachable: {{ $labels.instance }}"
|
|
description: "Service at {{ $labels.instance }} is not responding to HTTP health checks for 3 minutes."
|
|
|
|
- name: service_health
|
|
interval: 60s
|
|
rules:
|
|
# Service responding but very slow
|
|
- alert: SlowHTTPResponse
|
|
expr: probe_duration_seconds{job=~"http-.*"} > 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: container
|
|
annotations:
|
|
summary: "Slow response: {{ $labels.instance }}"
|
|
description: "{{ $labels.instance }} is responding but taking more than 10 seconds."
|
|
|
|
# Prometheus scrape target down
|
|
- alert: PrometheusTargetDown
|
|
expr: up == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: monitoring
|
|
annotations:
|
|
summary: "Prometheus target down: {{ $labels.job }}"
|
|
description: "Prometheus cannot scrape {{ $labels.instance }} (job: {{ $labels.job }}) for 5 minutes."
|
|
|
|
# Alerting pipeline dead-man's switch (Watchdog always fires - if it stops, pipeline is broken)
|
|
- alert: Watchdog
|
|
expr: vector(1)
|
|
labels:
|
|
severity: none
|
|
type: monitoring
|
|
annotations:
|
|
summary: "Alerting pipeline active"
|
|
description: "This is a dead-man's-switch alert. If you stop receiving this, the alerting pipeline is broken."
|
|
|
|
- name: frigate_alerts
|
|
interval: 60s
|
|
rules:
|
|
# Frigate high memory (standalone container on hf-pdocker-01 with host network)
|
|
- alert: FrigateHighMemory
|
|
expr: container_memory_usage_bytes{name="frigate"} / container_spec_memory_limit_bytes{name="frigate"} > 0.85
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: frigate
|
|
annotations:
|
|
summary: "Frigate high memory usage"
|
|
description: "Frigate is using more than 85% of allocated memory."
|
|
|
|
- name: service_resources
|
|
interval: 60s
|
|
rules:
|
|
# Any service container using more than 90% of its memory limit
|
|
- alert: ServiceHighMemory
|
|
expr: >
|
|
(container_memory_usage_bytes{name!="",name!~".*_db.*",name!~".*db$"} /
|
|
container_spec_memory_limit_bytes{name!="",name!~".*_db.*",name!~".*db$"} > 0)
|
|
> 0.90
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
type: container
|
|
annotations:
|
|
summary: "{{ $labels.name }}: High memory usage"
|
|
description: "Container {{ $labels.name }} on {{ $labels.instance }} is using more than 90% of its memory limit."
|