hobbs-iac/nomad/jobs/monitoring.nomad

job "monitoring" {
  datacenters = ["hf-dc1"]
  type        = "service"

  group "prometheus" {
    count = 1

    network {
      port "prometheus" {
        static = 9095
      }
    }

    service {
      name = "prometheus"
      port = "prometheus"
      
      tags = [
        "traefik.enable=true",
        "traefik.http.routers.prometheus.rule=Host(`prometheus.hobbs.farm`)",
      ]

      check {
        type     = "http"
        path     = "/-/healthy"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "prometheus" {
      driver = "docker"

      config {
        image = "prom/prometheus:v2.48.0"
        ports = ["prometheus"]
        
        args = [
          "--config.file=/etc/prometheus/prometheus.yml",
          "--storage.tsdb.path=/prometheus",
          "--web.listen-address=:9095",
          "--web.enable-lifecycle",
        ]
      }

      template {
        data = <<-EOF
          global:
            scrape_interval: 15s
            evaluation_interval: 15s
          
          scrape_configs:
            - job_name: 'prometheus'
              static_configs:
                - targets: ['localhost:9095']
            
            - job_name: 'nomad'
              consul_sd_configs:
                - server: '{{ env "NOMAD_IP_prometheus" }}:8500'
                  services: ['nomad-client', 'nomad']
              relabel_configs:
                - source_labels: ['__meta_consul_tags']
                  regex: '.*,metrics,.*'
                  action: keep
            
            - job_name: 'consul'
              static_configs:
                - targets: ['{{ env "NOMAD_IP_prometheus" }}:8500']
        EOF
        
        destination = "local/prometheus.yml"
      }

      resources {
        cpu    = 200
        memory = 512
      }

      volume_mount {
        volume      = "docker-data"
        destination = "/prometheus"
        read_only   = false
      }
    }

    volume "docker-data" {
      type      = "host"
      source    = "docker-data"
      read_only = false
    }
  }

  group "grafana" {
    count = 1

    network {
      port "grafana" {
        static = 3001
      }
    }

    service {
      name = "grafana"
      port = "grafana"
      
      tags = [
        "traefik.enable=true",
        "traefik.http.routers.grafana.rule=Host(`grafana.hobbs.farm`)",
      ]

      check {
        type     = "http"
        path     = "/api/health"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "grafana" {
      driver = "docker"

      config {
        image = "grafana/grafana:10.2.2"
        ports = ["grafana"]
      }

      env {
        GF_SERVER_HTTP_PORT = "3001"
        GF_SECURITY_ADMIN_PASSWORD = "admin"  # TODO: Use Vault
      }

      resources {
        cpu    = 200
        memory = 256
      }

      volume_mount {
        volume      = "docker-data"
        destination = "/var/lib/grafana"
        read_only   = false
      }
    }

    volume "docker-data" {
      type      = "host"
      source    = "docker-data"
      read_only = false
    }
  }

  group "alertmanager" {
    count = 1

    network {
      port "alertmanager" {
        static = 9093
      }
    }

    service {
      name = "alertmanager"
      port = "alertmanager"

      check {
        type     = "http"
        path     = "/-/healthy"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "alertmanager" {
      driver = "docker"

      config {
        image = "prom/alertmanager:v0.26.0"
        ports = ["alertmanager"]
      }

      resources {
        cpu    = 100
        memory = 128
      }
    }
  }
}