hobbs-iac/stacks/monitoring/alerts.yml

groups:
  - name: infrastructure_devices
    interval: 30s
    rules:
      # Unifi Device Down
      - alert: UnifiDeviceDown
        expr: unifi_device_up == 0
        for: 2m
        labels:
          severity: critical
          type: infrastructure
        annotations:
          summary: "Unifi device {{ $labels.name }} is offline"
          description: "Device {{ $labels.name }} ({{ $labels.model }}) at {{ $labels.ip }} has been offline for more than 2 minutes."

  - name: server_health
    interval: 60s
    rules:
      # Server Down
      - alert: ServerDown
        expr: up{job="node"} == 0
        for: 1m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "Server {{ $labels.hostname }} is DOWN"
          description: "{{ $labels.hostname }} ({{ $labels.host_ip }}) is not responding to health checks for 1 minute."

      # High disk usage
      - alert: HighDiskUsage
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
        for: 5m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: Disk space critically low"
          description: "Root filesystem on {{ $labels.hostname }} has less than 10% free space remaining."

      # Moderate disk usage warning
      - alert: DiskSpaceWarning
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 20
        for: 10m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: Disk space low"
          description: "Root filesystem on {{ $labels.hostname }} has less than 20% free space."

      # High memory usage
      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
        for: 5m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: High memory usage"
          description: "Memory usage on {{ $labels.hostname }} is above 90% for 5 minutes."

      # Critical memory usage
      - alert: CriticalMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 95
        for: 2m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: Critical memory usage"
          description: "Memory usage on {{ $labels.hostname }} is above 95%. System may become unstable."

      # High CPU usage (bart: 90% threshold due to Frigate/media workloads, others: 80%)
      - alert: HighCPUUsage
        expr: >
          (100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname!="bart"}[5m])) * 100) > 80)
          or
          (100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle",hostname="bart"}[5m])) * 100) > 90)
        for: 10m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: High CPU usage"
          description: "CPU usage on {{ $labels.hostname }} is above threshold for 10 minutes."

      # Critical CPU usage
      - alert: CriticalCPUUsage
        expr: 100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
        for: 5m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: Critical CPU usage"
          description: "CPU usage on {{ $labels.hostname }} is above 95% for 5 minutes. Performance degraded."

      # High system load
      - alert: HighSystemLoad
        expr: node_load5 / on(hostname) count by(hostname) (node_cpu_seconds_total{mode="idle"}) > 2
        for: 10m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: High system load"
          description: "5-minute load average on {{ $labels.hostname }} is more than 2x CPU count."

      # Server rebooted
      - alert: ServerRebooted
        expr: node_time_seconds - node_boot_time_seconds < 600
        for: 1m
        labels:
          severity: info
          type: server
        annotations:
          summary: "{{ $labels.hostname }} rebooted"
          description: "{{ $labels.hostname }} was rebooted less than 10 minutes ago."

      # High swap usage
      - alert: HighSwapUsage
        expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 50)
        for: 5m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: High swap usage"
          description: "Swap usage on {{ $labels.hostname }} is above 50% for 5 minutes. Check container memory limits."

      # Critical swap usage
      - alert: CriticalSwapUsage
        expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100 > 80)
        for: 2m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: Critical swap usage"
          description: "Swap usage on {{ $labels.hostname }} is above 80%. System may become unresponsive."

      # NFS mount failure (all data is on NAS - this is critical)
      - alert: NFSMountDown
        expr: node_filesystem_avail_bytes{fstype="nfs"} == 0 or absent(node_filesystem_avail_bytes{fstype="nfs"})
        for: 1m
        labels:
          severity: critical
          type: server
        annotations:
          summary: "{{ $labels.hostname }}: NFS mount unavailable"
          description: "NFS mount on {{ $labels.hostname }} at {{ $labels.mountpoint }} is down. All services depend on NAS storage."

      # NAS disk space low
      - alert: NASDiskSpaceLow
        expr: (node_filesystem_avail_bytes{fstype="nfs",mountpoint="/mnt/docker-data"} / node_filesystem_size_bytes{fstype="nfs",mountpoint="/mnt/docker-data"}) * 100 < 15
        for: 10m
        labels:
          severity: warning
          type: server
        annotations:
          summary: "NAS disk space low"
          description: "NFS share /mnt/docker-data has less than 15% free space."

  - name: docker_containers
    interval: 60s
    rules:
      # Service/container unreachable via blackbox HTTP probe
      # Matches all http-pdocker01, http-pdocker02, http-bart, and http-external jobs
      - alert: ContainerDown
        expr: probe_success{job=~"http-.*"} == 0
        for: 3m
        labels:
          severity: warning
          type: container
        annotations:
          summary: "Service unreachable: {{ $labels.instance }}"
          description: "Service at {{ $labels.instance }} is not responding to HTTP health checks for 3 minutes."

  - name: service_health
    interval: 60s
    rules:
      # Service responding but very slow
      - alert: SlowHTTPResponse
        expr: probe_duration_seconds{job=~"http-.*"} > 10
        for: 5m
        labels:
          severity: warning
          type: container
        annotations:
          summary: "Slow response: {{ $labels.instance }}"
          description: "{{ $labels.instance }} is responding but taking more than 10 seconds."

      # Prometheus scrape target down
      - alert: PrometheusTargetDown
        expr: up == 0
        for: 5m
        labels:
          severity: warning
          type: monitoring
        annotations:
          summary: "Prometheus target down: {{ $labels.job }}"
          description: "Prometheus cannot scrape {{ $labels.instance }} (job: {{ $labels.job }}) for 5 minutes."

      # Alerting pipeline dead-man's switch (Watchdog always fires - if it stops, pipeline is broken)
      - alert: Watchdog
        expr: vector(1)
        labels:
          severity: none
          type: monitoring
        annotations:
          summary: "Alerting pipeline active"
          description: "This is a dead-man's-switch alert. If you stop receiving this, the alerting pipeline is broken."

  - name: frigate_alerts
    interval: 60s
    rules:
      # Frigate high memory (standalone container on hf-pdocker-01 with host network)
      - alert: FrigateHighMemory
        expr: container_memory_usage_bytes{name="frigate"} / container_spec_memory_limit_bytes{name="frigate"} > 0.85
        for: 5m
        labels:
          severity: warning
          type: frigate
        annotations:
          summary: "Frigate high memory usage"
          description: "Frigate is using more than 85% of allocated memory."

  - name: service_resources
    interval: 60s
    rules:
      # Any service container using more than 90% of its memory limit
      - alert: ServiceHighMemory
        expr: >
          (container_memory_usage_bytes{name!="",name!~".*_db.*",name!~".*db$"} /
           container_spec_memory_limit_bytes{name!="",name!~".*_db.*",name!~".*db$"} > 0)
          > 0.90
        for: 5m
        labels:
          severity: warning
          type: container
        annotations:
          summary: "{{ $labels.name }}: High memory usage"
          description: "Container {{ $labels.name }} on {{ $labels.instance }} is using more than 90% of its memory limit."