hobbs-iac/stacks/monitoring/prometheus.yml

global:
  scrape_interval: 1m
  evaluation_interval: 1m

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 'alertmanager:9093'

rule_files:
  - "alerts.yml"

scrape_configs:
  # Prometheus self-monitoring
  - job_name: 'prometheus'
    scrape_interval: 1m
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter (system metrics) - all 3 swarm nodes
  - job_name: 'node'
    static_configs:
      - targets: ['192.168.86.192:9100']
        labels:
          hostname: 'hf-pdocker-01'
          host_ip: '192.168.86.192'
      - targets: ['192.168.86.167:9100']
        labels:
          hostname: 'bart'
          host_ip: '192.168.86.167'
      - targets: ['192.168.86.100:9100']
        labels:
          hostname: 'hf-pdocker-02'
          host_ip: '192.168.86.100'

  # Blackbox Exporter self-monitoring
  - job_name: 'blackbox'
    static_configs:
      - targets: ['blackbox-exporter:9115']

  # Matrix Alertmanager Receiver metrics (now on hf-pdocker-01)
  - job_name: 'matrix-alertmanager-receiver'
    scrape_interval: 1m
    static_configs:
      - targets: ['192.168.86.192:9094']
        labels:
          hostname: 'hf-pdocker-01'

  # HTTP Health Checks - hf-pdocker-01 services
  - job_name: 'http-pdocker01'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - http://192.168.86.192:5000        # Frigate NVR (standalone, host network)
        - http://192.168.86.192:5199/health  # Frigate Notify
        - http://192.168.86.192:5200/health  # clip-service (CLIP embeddings)
        - http://192.168.86.192:8888/health  # Farm Manager API
        - http://192.168.86.192:8889/health  # Farm Agent
        - http://192.168.86.192:8123        # Home Assistant
        - http://192.168.86.192:6052        # ESPHome
        - http://192.168.86.192:8199/guacamole/  # Guacamole
        - http://192.168.86.192:8008        # Matrix Synapse
        - http://192.168.86.192:3000        # AdGuard Home
        - http://192.168.86.192:8090/dashboard/  # Traefik Dashboard
        - http://192.168.86.192:3001        # Grafana
        - http://192.168.86.192:9095        # Prometheus
        - http://192.168.86.192:9093        # Alertmanager
        - http://192.168.86.192:18795       # Spirit (infra ops agent)
        - http://192.168.86.192:8085        # VoiceCoach web client
        - http://192.168.86.192:8181/health  # Rongorongo (probe internal — Traefik basicAuth blocks external)
        - http://192.168.86.192:7880        # VoiceCoach LiveKit server
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  # HTTP Health Checks - hf-pdocker-02 services
  - job_name: 'http-pdocker02'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - http://192.168.86.100:3003        # Gitea
        - http://192.168.86.100:3004        # Semaphore
        - http://192.168.86.100:3002        # Wiki.js
        - http://192.168.86.100:26226       # Vaultwarden
        - http://192.168.86.100:5006        # Actual Budget
        - http://192.168.86.100:8001/openapi.json  # MCPO
        - http://192.168.86.100:8889/health  # Farm Agent
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  # HTTPS Health Checks - services with self-signed certs
  - job_name: 'http-pdocker02-https'
    metrics_path: /probe
    params:
      module: [http_2xx_insecure]
    static_configs:
      - targets:
        - https://192.168.86.100:8443       # Unifi Controller (self-signed cert)
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  # HTTP Health Checks - bart services
  - job_name: 'http-bart'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - http://192.168.86.167:8096        # Jellyfin
        - http://192.168.86.167:8989        # Sonarr
        - http://192.168.86.167:7878        # Radarr
        - http://192.168.86.167:8082        # SABnzbd
        - http://192.168.86.167:5076        # NZBHydra2
        - http://192.168.86.167:5055        # Jellyseerr
        - http://192.168.86.167:9392        # OpenVAS
        - http://192.168.86.167:8889/health  # Farm Agent
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  # HTTPS Health Checks - External domains via Traefik
  - job_name: 'http-external'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://mess.hobbs.farm          # Matrix (via Traefik)
        - https://solo.hobbs.farm/guacamole/  # Guacamole (via Traefik)
        - https://yoda.hobbs.farm          # Home Assistant
        - https://vault.hobbs.farm         # Vaultwarden
        - https://git.hobbs.farm           # Gitea
        - https://coach.hobbs.farm         # VoiceCoach web client
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  # Unifi Network Devices - uncomment when unifi-exporter is deployed
  # - job_name: 'unifi-devices'
  #   scrape_interval: 30s
  #   static_configs:
  #     - targets: ['unifi-exporter:9101']