# llama.cpp server on .150 (Docker Desktop) — inference backend for hermes/FRIDAY.
#
# Replaced ollama as the *serving* engine on 2026-08-21 after a head-to-head
# benchmark on the same GGUF, same 64k context, same GPU:
#   ollama (f16 KV):    15.8 tok/s aggregate, 16.5 sustained, 23207 MiB VRAM
#   llama.cpp (q8_0 KV): 39.7 tok/s aggregate, 39.9 sustained, 19545 MiB VRAM
# The difference is the quantized KV cache: it drops the footprint from 23.2GB
# to 18.6GB, so the whole model fits on the 3090 instead of spilling layers to
# CPU. Ollama exposes no knob for KV quantization.
#
# Image is pinned BY DIGEST on purpose — `:server-cuda` is a moving tag, and
# this homelab has been bitten by tag-rot three times (synapse, gitea, openclaw).
services:
  llamacpp:
    image: ghcr.io/ggml-org/llama.cpp@sha256:995fe9f237c622ac257905948c3cdde447ab343db22e0686322445b5ec6d3131
    container_name: llamacpp
    restart: unless-stopped
    ports:
      - "8081:8081"
    volumes:
      # Read-only mount of ollama's blob store. NOTE the coupling: this is the
      # GGUF that `qwen3.8` resolves to. C: only had 26GB free so a dedicated
      # copy wasn't practical. If you ever `ollama rm qwen3.8`, this breaks —
      # copy the blob out first.
      - "C:/Users/Jeremy/.ollama/models/blobs:/models:ro"
    command:
      - "-m"
      - "/models/sha256-f5f1dd8920d417aac2718b0bda3403da274301efdd6760b4f0f4b864ff2ad57d"
      - "--alias"
      - "qwen3.8-64k"
      - "-ngl"
      - "99"                      # all 65 layers to GPU
      - "-c"
      - "65536"                   # hermes hard-refuses anything under 64K
      - "--cache-type-k"
      - "q8_0"                    # the whole point — halves KV cache
      - "--cache-type-v"
      - "q8_0"
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "8081"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    labels:
      - "com.centurylinklabs.watchtower.enable=false"
    # The image ships its own HEALTHCHECK that cannot run here, so the
    # container reports permanently "unhealthy" while serving fine (cf. litellm,
    # CLAUDE.md known issue #15). Disable it; liveness is the blackbox probe of
    # GET http://192.168.86.150:8081/health from outside.
    healthcheck:
      disable: true
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "3"