pluto
2df137ee2a
- OllamaClient: analyze_image, compare_images, is_available - Recognition pipeline: CLIP embedding -> cosine similarity -> threshold routing (known >0.85, unknown <0.60, ambiguous -> Ollama fallback) - Vision dispatcher: get_description() supports haiku/ollama/both backends - 9 embedding serialization tests Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
133 lines
4.7 KiB
Python
133 lines
4.7 KiB
Python
"""Ollama vision LLM client for frigate-notify v2."""
|
|
|
|
import base64
|
|
import logging
|
|
|
|
import aiohttp
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class OllamaClient:
|
|
"""Client for Ollama vision models (e.g., llava)."""
|
|
|
|
def __init__(self, base_url: str, model: str):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.model = model
|
|
|
|
async def analyze_image(self, image_bytes: bytes, prompt: str) -> str | None:
|
|
"""Send an image to Ollama for analysis.
|
|
|
|
Args:
|
|
image_bytes: Raw image bytes (JPEG).
|
|
prompt: Text prompt for the vision model.
|
|
|
|
Returns:
|
|
Response text or None on failure.
|
|
"""
|
|
b64 = base64.b64encode(image_bytes).decode("utf-8")
|
|
payload = {
|
|
"model": self.model,
|
|
"prompt": prompt,
|
|
"images": [b64],
|
|
"stream": False,
|
|
"options": {"num_predict": 200},
|
|
}
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(
|
|
f"{self.base_url}/api/generate",
|
|
json=payload,
|
|
timeout=aiohttp.ClientTimeout(total=30),
|
|
) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
text = data.get("response", "").strip()
|
|
logger.info(
|
|
"Ollama analysis complete: %d chars", len(text)
|
|
)
|
|
return text if text else None
|
|
body = await resp.text()
|
|
logger.error(
|
|
"Ollama analyze_image failed %s: %s", resp.status, body
|
|
)
|
|
except Exception:
|
|
logger.exception("Ollama analyze_image request failed")
|
|
return None
|
|
|
|
async def compare_images(
|
|
self,
|
|
image_bytes: bytes,
|
|
reference_bytes: bytes,
|
|
object_name: str,
|
|
) -> bool | None:
|
|
"""Compare two images to determine if they show the same object/person.
|
|
|
|
Args:
|
|
image_bytes: The new event image bytes.
|
|
reference_bytes: The reference (known) image bytes.
|
|
object_name: Name of the known object for context.
|
|
|
|
Returns:
|
|
True if same, False if different, None on failure/inconclusive.
|
|
"""
|
|
b64_new = base64.b64encode(image_bytes).decode("utf-8")
|
|
b64_ref = base64.b64encode(reference_bytes).decode("utf-8")
|
|
prompt = (
|
|
f"Look at these two images. The first image is from a security camera. "
|
|
f"The second image is a reference photo of '{object_name}'. "
|
|
f"Are these the same person/vehicle/animal? "
|
|
f"Answer with ONLY 'yes' or 'no'."
|
|
)
|
|
payload = {
|
|
"model": self.model,
|
|
"prompt": prompt,
|
|
"images": [b64_new, b64_ref],
|
|
"stream": False,
|
|
"options": {"num_predict": 200},
|
|
}
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(
|
|
f"{self.base_url}/api/generate",
|
|
json=payload,
|
|
timeout=aiohttp.ClientTimeout(total=30),
|
|
) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
text = data.get("response", "").strip().lower()
|
|
logger.info("Ollama compare result: %s", text)
|
|
if "yes" in text.split()[:3]:
|
|
return True
|
|
if "no" in text.split()[:3]:
|
|
return False
|
|
logger.warning(
|
|
"Ollama compare inconclusive: %s", text
|
|
)
|
|
return None
|
|
body = await resp.text()
|
|
logger.error(
|
|
"Ollama compare_images failed %s: %s",
|
|
resp.status,
|
|
body,
|
|
)
|
|
except Exception:
|
|
logger.exception("Ollama compare_images request failed")
|
|
return None
|
|
|
|
async def is_available(self) -> bool:
|
|
"""Check if Ollama is reachable and responding.
|
|
|
|
Returns:
|
|
True if Ollama responds with HTTP 200, False otherwise.
|
|
"""
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(
|
|
f"{self.base_url}/api/tags",
|
|
timeout=aiohttp.ClientTimeout(total=3),
|
|
) as resp:
|
|
return resp.status == 200
|
|
except Exception:
|
|
return False
|