feat: add recognition engine with CLIP matching and Ollama vision LLM

- OllamaClient: analyze_image, compare_images, is_available
- Recognition pipeline: CLIP embedding -> cosine similarity -> threshold
  routing (known >0.85, unknown <0.60, ambiguous -> Ollama fallback)
- Vision dispatcher: get_description() supports haiku/ollama/both backends
- 9 embedding serialization tests

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-07 17:17:49 -06:00
parent 74e12eca60
commit 2df137ee2a
4 changed files with 513 additions and 0 deletions
+67
View File
@@ -1,7 +1,11 @@
import base64
import logging
from typing import Optional
import anthropic
from ollama_client import OllamaClient
logger = logging.getLogger(__name__)
PROMPT = (
@@ -40,3 +44,66 @@ def analyze_snapshot(client: anthropic.Anthropic, image_bytes: bytes) -> str | N
except Exception:
logger.exception("Claude Vision API failed")
return None
async def analyze_snapshot_ollama(
ollama_client: OllamaClient, image_bytes: bytes
) -> str | None:
"""Send a snapshot to Ollama vision model for description.
Args:
ollama_client: Configured OllamaClient instance.
image_bytes: Raw image bytes (JPEG).
Returns:
Description string or None on failure.
"""
return await ollama_client.analyze_image(image_bytes, PROMPT)
async def get_description(
image_bytes: bytes,
vision_backend: str,
claude_client: Optional[anthropic.Anthropic] = None,
ollama_client: Optional[OllamaClient] = None,
) -> str | None:
"""Get a scene description using the configured vision backend.
Args:
image_bytes: Raw image bytes (JPEG).
vision_backend: One of "haiku", "ollama", or "both".
"both" tries Ollama first, falls back to Haiku.
claude_client: Anthropic client (required for "haiku" and "both").
ollama_client: OllamaClient instance (required for "ollama" and "both").
Returns:
Description string or None if all backends fail.
"""
if vision_backend == "ollama":
if ollama_client is None:
logger.error("Ollama backend selected but no client provided")
return None
return await analyze_snapshot_ollama(ollama_client, image_bytes)
if vision_backend == "haiku":
if claude_client is None:
logger.error("Haiku backend selected but no client provided")
return None
return analyze_snapshot(claude_client, image_bytes)
if vision_backend == "both":
# Try Ollama first, fall back to Haiku
if ollama_client is not None:
result = await analyze_snapshot_ollama(ollama_client, image_bytes)
if result is not None:
return result
logger.info("Ollama failed, falling back to Haiku")
if claude_client is not None:
return analyze_snapshot(claude_client, image_bytes)
logger.error("Both backend selected but no clients provided")
return None
logger.error("Unknown vision backend: %s", vision_backend)
return None