jeremy/farm-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: agent timeouts hid hf-pdocker-01; refresh links and groups

Browse Source 
- agent: stats one_shot=True + 20 workers (.192 went 12s -> ~2s);
  /health ignores Dead containers
- server: /containers fetch timeout 10s -> 30s (was silently returning
  [] for the busiest node)
- links page: drop FRIDAY Voice, add Onyx, Guacamole, Synapse Admin,
  n8n, CouchDB; MCPO -> /docs, Trivy -> /healthz, Jellyseerr -> Seerr
- config.json: drop dead gpu-workstation (192.168.86.27)
- groups.json: rebuilt for current container names (14 groups)
- includes earlier uncommitted work (models, styles, ports sorting)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NQhgEUqcaV4WM2S4hfXgHD
This commit is contained in:
Pluto
2026-10-26 16:27:43 -05:00
parent 6338621ad4
commit 446c01fb78
8 changed files with 559 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/docker_ops.py
+83 -2
View File
@@ -1,6 +1,7 @@
"""Docker operations wrapper for managing Swarm and Compose containers."""
import socket
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import docker
@@ -77,6 +78,51 @@ class DockerOps:
except Exception:
return "unknown"
@staticmethod
def _extract_ports(container) -> list[str]:
"""Extract exposed host ports from a container's network settings.
Returns a sorted, deduplicated list of strings like '8080', '443'.
Only includes ports bound to the host. For host-network containers,
reads the exposed ports from the image config instead.
"""
ports = set()
network_mode = container.attrs.get("HostConfig", {}).get("NetworkMode", "")
if network_mode == "host":
# Host networking: use exposed ports from the image/container config
exposed = container.attrs.get("Config", {}).get("ExposedPorts", {})
for port_proto in exposed:
port = port_proto.split("/")[0]
ports.add(port)
else:
# Bridge/overlay: use published port bindings
port_bindings = container.attrs.get("NetworkSettings", {}).get("Ports", {}) or {}
for port_proto, bindings in port_bindings.items():
if bindings:
for binding in bindings:
host_port = binding.get("HostPort")
if host_port:
ports.add(host_port)
return sorted(ports, key=lambda p: int(p) if p.isdigit() else 0)
def _get_container_memory(self, container_id: str) -> tuple[str, int, int]:
"""Fetch memory usage and limit for a single running container.
Returns (container_id, usage_bytes, limit_bytes).
"""
try:
stats = self.client.api.stats(container_id, stream=False, one_shot=True)
mem = stats.get("memory_stats", {})
usage = mem.get("usage", 0)
limit = mem.get("limit", 0)
# Subtract inactive file cache for accurate usage
cache = mem.get("stats", {}).get("inactive_file", 0)
return (container_id, max(usage - cache, 0), limit)
except Exception:
return (container_id, 0, 0)
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
@@ -89,7 +135,7 @@ class DockerOps:
Returns a list of dicts, each containing:
id, name, status, image, created, uptime, is_swarm,
swarm_service
swarm_service, memory_usage, memory_limit
Uses the low-level API to get container IDs to avoid NotFound errors
from containers that disappear between list and inspect (e.g. Swarm
@@ -98,6 +144,7 @@ class DockerOps:
raw_ids = [r["Id"] for r in self.client.api.containers(all=True)]
result = []
swarm_services_with_containers = set()
running_ids = {} # full_id -> short_id for memory stats lookup
for cid in raw_ids:
try:
@@ -110,6 +157,8 @@ class DockerOps:
if swarm_service:
swarm_services_with_containers.add(swarm_service)
started_at = c.attrs.get("State", {}).get("StartedAt", "")
if c.status == "running":
running_ids[cid] = c.short_id
result.append(
{
"id": c.short_id,
@@ -120,9 +169,28 @@ class DockerOps:
"uptime": self._format_uptime(started_at) if c.status == "running" else None,
"is_swarm": self._is_swarm(c),
"swarm_service": swarm_service,
"ports": self._extract_ports(c),
"memory_usage": 0,
"memory_limit": 0,
}
)
# Fetch memory stats for running containers in parallel
if running_ids:
mem_stats = {}
with ThreadPoolExecutor(max_workers=20) as pool:
futures = {
pool.submit(self._get_container_memory, full_id): short_id
for full_id, short_id in running_ids.items()
}
for future in futures:
full_id, usage, limit = future.result()
short_id = futures[future]
mem_stats[short_id] = (usage, limit)
for entry in result:
if entry["id"] in mem_stats:
entry["memory_usage"], entry["memory_limit"] = mem_stats[entry["id"]]
# Include Swarm services with 0 replicas (only works on manager nodes)
try:
for svc in self.client.services.list():
@@ -135,6 +203,15 @@ class DockerOps:
)
if "@" in image:
image = image.split("@")[0]
# Extract published ports from Swarm service endpoint spec
svc_ports = []
endpoint_spec = svc.attrs.get("Spec", {}).get("EndpointSpec", {})
for p in endpoint_spec.get("Ports", []):
published = p.get("PublishedPort")
if published:
svc_ports.append(str(published))
svc_ports.sort(key=lambda x: int(x) if x.isdigit() else 0)
result.append(
{
"id": f"swarm:{svc.name}",
@@ -145,6 +222,7 @@ class DockerOps:
"uptime": None,
"is_swarm": True,
"swarm_service": svc.name,
"ports": svc_ports,
}
)
except Exception:
@@ -336,7 +414,10 @@ class DockerOps:
Uses the low-level API to avoid NotFound errors from containers
stuck in Docker's 'Dead' state.
"""
containers = self.client.api.containers(all=True)
containers = [
c for c in self.client.api.containers(all=True)
if c.get("State") != "dead"
]
return {
"status": "healthy",
"hostname": socket.gethostname(),