jeremy
f2374506b4
- hermes: pin nousresearch/hermes-agent:v2026.8.18 (was :latest, a rolling dev build that Watchtower never re-pulled -> 7 weeks stale). - llamacpp: add compose for the llama.cpp inference backend serving hermes on .150:8081 (q8_0 KV cache; 2.4x ollama on the same GGUF). - ai-agents: add reconstructed, version-pinned Dockerfiles for margaret and spirit (openclaw was built from @latest with no Dockerfile stored). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
63 lines
2.4 KiB
YAML
63 lines
2.4 KiB
YAML
# llama.cpp server on .150 (Docker Desktop) — inference backend for hermes/FRIDAY.
|
|
#
|
|
# Replaced ollama as the *serving* engine on 2026-08-21 after a head-to-head
|
|
# benchmark on the same GGUF, same 64k context, same GPU:
|
|
# ollama (f16 KV): 15.8 tok/s aggregate, 16.5 sustained, 23207 MiB VRAM
|
|
# llama.cpp (q8_0 KV): 39.7 tok/s aggregate, 39.9 sustained, 19545 MiB VRAM
|
|
# The difference is the quantized KV cache: it drops the footprint from 23.2GB
|
|
# to 18.6GB, so the whole model fits on the 3090 instead of spilling layers to
|
|
# CPU. Ollama exposes no knob for KV quantization.
|
|
#
|
|
# Image is pinned BY DIGEST on purpose — `:server-cuda` is a moving tag, and
|
|
# this homelab has been bitten by tag-rot three times (synapse, gitea, openclaw).
|
|
services:
|
|
llamacpp:
|
|
image: ghcr.io/ggml-org/llama.cpp@sha256:995fe9f237c622ac257905948c3cdde447ab343db22e0686322445b5ec6d3131
|
|
container_name: llamacpp
|
|
restart: unless-stopped
|
|
ports:
|
|
- "8081:8081"
|
|
volumes:
|
|
# Read-only mount of ollama's blob store. NOTE the coupling: this is the
|
|
# GGUF that `qwen3.8` resolves to. C: only had 26GB free so a dedicated
|
|
# copy wasn't practical. If you ever `ollama rm qwen3.8`, this breaks —
|
|
# copy the blob out first.
|
|
- "C:/Users/Jeremy/.ollama/models/blobs:/models:ro"
|
|
command:
|
|
- "-m"
|
|
- "/models/sha256-f5f1dd8920d417aac2718b0bda3403da274301efdd6760b4f0f4b864ff2ad57d"
|
|
- "--alias"
|
|
- "qwen3.8-64k"
|
|
- "-ngl"
|
|
- "99" # all 65 layers to GPU
|
|
- "-c"
|
|
- "65536" # hermes hard-refuses anything under 64K
|
|
- "--cache-type-k"
|
|
- "q8_0" # the whole point — halves KV cache
|
|
- "--cache-type-v"
|
|
- "q8_0"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8081"
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: 1
|
|
capabilities: [gpu]
|
|
labels:
|
|
- "com.centurylinklabs.watchtower.enable=false"
|
|
# The image ships its own HEALTHCHECK that cannot run here, so the
|
|
# container reports permanently "unhealthy" while serving fine (cf. litellm,
|
|
# CLAUDE.md known issue #15). Disable it; liveness is the blackbox probe of
|
|
# GET http://192.168.86.150:8081/health from outside.
|
|
healthcheck:
|
|
disable: true
|
|
logging:
|
|
driver: "json-file"
|
|
options:
|
|
max-size: "50m"
|
|
max-file: "3"
|