Compare commits

...

6 Commits

Author SHA1 Message Date
jeremy ff5d7b7841 fix: correct JavaScript field names to match API response schema
The web UI was referencing stale field names that didn't match the
actual API response fields returned by the backend:

  event.status         -> event.match_status
  event.identified_as  -> event.matched_object_name
  event.description    -> event.ai_description
  event.timestamp      -> event.created_at
  stats.known          -> stats.known_matches
  stats.unknown        -> stats.unknown_events
  stats.known_objects  -> stats.known_objects_count

This caused the review queue to display blank badges, timestamps,
descriptions, and identifications, and the dashboard stats to show
all zeros.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-08 20:19:42 -05:00
pluto 85ecbc22ac feat: add automatic event cleanup based on retention setting
Hourly background task deletes events older than configured
retention period (default 90 days). Reads event_retention_days
from settings table. Includes 2 tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 17:24:08 -06:00
pluto 5908293671 feat: add web UI with dashboard, review queue, known objects, and settings
Vanilla HTML/CSS/JS SPA served at :5199 with dark theme:
- Dashboard: live event feed with snapshot thumbnails, filters, auto-refresh
- Review Queue: unreviewed events with tag/dismiss actions
- Known Objects: grid of tagged identities with delete
- Settings: vision backend toggle, recognition enable, retention days
- Tag modal: assign events to existing or new known objects

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 17:24:02 -06:00
pluto f5f688277f feat: integrate recognition engine into MQTT event pipeline
- Recognition pipeline runs between snapshot fetch and notification
- Checks recognition_enabled setting before running CLIP matching
- Known objects with quiet/suppress action skip Matrix notifications
- Vision description uses configurable backend (haiku/ollama/both)
- docker-compose.yml adds CLIP/Ollama env vars and data volume

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 17:23:54 -06:00
pluto 2df137ee2a feat: add recognition engine with CLIP matching and Ollama vision LLM
- OllamaClient: analyze_image, compare_images, is_available
- Recognition pipeline: CLIP embedding -> cosine similarity -> threshold
  routing (known >0.85, unknown <0.60, ambiguous -> Ollama fallback)
- Vision dispatcher: get_description() supports haiku/ollama/both backends
- 9 embedding serialization tests

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 17:17:49 -06:00
pluto 74e12eca60 feat: add CLIP embedding service sidecar for GPU machine
FastAPI service that loads ViT-L-14 model and exposes /embed,
/similarity, and /batch-similarity endpoints. Runs on RTX 3090
with NVIDIA GPU reservation. Includes 9 cosine similarity tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 17:17:42 -06:00
18 changed files with 2303 additions and 10 deletions
+35
View File
@@ -0,0 +1,35 @@
"""Automatic event cleanup based on retention settings."""
import asyncio
import logging
from database import get_connection
logger = logging.getLogger(__name__)
async def cleanup_old_events():
"""Delete events older than the configured retention period. Runs hourly."""
while True:
try:
conn = get_connection()
row = conn.execute(
"SELECT value FROM settings WHERE key='event_retention_days'"
).fetchone()
days = int(row["value"]) if row else 90
result = conn.execute(
"DELETE FROM events WHERE created_at < datetime('now', ? || ' days')",
(f"-{days}",),
)
conn.commit()
if result.rowcount > 0:
logger.info(
"Cleaned up %d events older than %d days",
result.rowcount,
days,
)
except Exception:
logger.exception("Cleanup failed")
await asyncio.sleep(3600)
+2 -1
View File
@@ -13,8 +13,9 @@ _db_path: str = "/data/frigate-notify.db"
def set_db_path(path: str) -> None:
"""Set the database path. Must be called before any other database operations."""
"""Set the database path. Closes any existing connection first."""
global _db_path
close_connection()
_db_path = path
+71 -9
View File
@@ -26,7 +26,10 @@ from matrix_client import MatrixClient
from routes.events import router as events_router
from routes.known_objects import router as known_objects_router
from routes.settings import router as settings_router
from vision import analyze_snapshot
from ollama_client import OllamaClient
from recognition import recognize_event, get_clip_embedding, embedding_to_bytes
from vision import get_description
from cleanup import cleanup_old_events
# --- Logging ---
logging.basicConfig(
@@ -67,7 +70,8 @@ async def fetch_snapshot(event_id: str) -> bytes | None:
# --- Event processing ---
async def process_event(
event_data: dict, claude: anthropic.Anthropic, matrix: MatrixClient
event_data: dict, claude: anthropic.Anthropic, matrix: MatrixClient,
ollama_client: OllamaClient = None,
):
global events_processed, last_event_time
@@ -117,10 +121,63 @@ async def process_event(
# --- Fetch snapshot ---
snapshot = await fetch_snapshot(event_id)
# --- AI analysis ---
# --- Recognition ---
match_status = "unreviewed"
matched_id = None
match_conf = 0.0
matched_name = None
# Check if recognition is enabled
recognition_enabled = False
try:
conn = get_connection()
row = conn.execute("SELECT value FROM settings WHERE key='recognition_enabled'").fetchone()
if row:
recognition_enabled = row["value"].lower() == "true"
except Exception:
pass
if recognition_enabled and snapshot:
match_status, matched_id, match_conf, matched_name = await recognize_event(
snapshot, ollama_client
)
# Store embedding with event
embedding = await get_clip_embedding(snapshot)
if embedding:
embedding_blob = embedding_to_bytes(embedding)
try:
conn = get_connection()
conn.execute(
"UPDATE events SET embedding=?, match_status=?, matched_object_id=?, match_confidence=? WHERE frigate_event_id=?",
(embedding_blob, match_status, matched_id, match_conf, event_id),
)
conn.commit()
except Exception:
logger.exception("Failed to update event recognition")
# If known, check action - skip notification for quiet/suppress
if match_status == "known" and matched_id:
conn = get_connection()
obj = conn.execute("SELECT action FROM known_objects WHERE id=?", (matched_id,)).fetchone()
if obj and obj["action"] in ("quiet", "suppress"):
logger.info("Known object '%s' (%s), skipping notification", matched_name, obj["action"])
events_processed += 1
last_event_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
return
# --- AI description ---
vision_backend = "haiku"
try:
conn = get_connection()
row = conn.execute("SELECT value FROM settings WHERE key='vision_backend'").fetchone()
if row:
vision_backend = row["value"]
except Exception:
pass
description = None
if snapshot:
description = analyze_snapshot(claude, snapshot)
description = await get_description(snapshot, vision_backend, claude, ollama_client)
if not description:
description = f"{label.capitalize()} detected on {camera}."
@@ -129,9 +186,9 @@ async def process_event(
conn = get_connection()
conn.execute(
"""INSERT OR IGNORE INTO events
(frigate_event_id, camera, label, score, snapshot, ai_description, notified)
VALUES (?, ?, ?, ?, ?, ?, 1)""",
(event_id, camera, label, score, snapshot, description),
(frigate_event_id, camera, label, score, snapshot, match_status, ai_description, notified)
VALUES (?, ?, ?, ?, ?, ?, ?, 1)""",
(event_id, camera, label, score, snapshot, match_status, description),
)
conn.commit()
except Exception:
@@ -175,6 +232,7 @@ def create_mqtt_client(
loop: asyncio.AbstractEventLoop,
claude: anthropic.Anthropic,
matrix: MatrixClient,
ollama_client: OllamaClient = None,
) -> mqtt.Client:
global mqtt_connected
@@ -193,7 +251,7 @@ def create_mqtt_client(
try:
payload = json.loads(msg.payload)
asyncio.run_coroutine_threadsafe(
process_event(payload, claude, matrix), loop
process_event(payload, claude, matrix, ollama_client), loop
)
except json.JSONDecodeError:
logger.warning("Invalid JSON on %s", msg.topic)
@@ -217,8 +275,9 @@ def start_mqtt(loop: asyncio.AbstractEventLoop) -> None:
config.MATRIX_ACCESS_TOKEN,
config.MATRIX_ROOM_ID,
)
ollama_client = OllamaClient(config.OLLAMA_URL, config.OLLAMA_MODEL)
mqtt_client_instance = create_mqtt_client(loop, claude, matrix)
mqtt_client_instance = create_mqtt_client(loop, claude, matrix, ollama_client)
mqtt_client_instance.connect(config.MQTT_HOST, config.MQTT_PORT)
thread = Thread(target=mqtt_client_instance.loop_forever, daemon=True)
@@ -240,6 +299,9 @@ async def lifespan(app: FastAPI):
loop = asyncio.get_event_loop()
start_mqtt(loop)
# Start periodic event cleanup
asyncio.create_task(cleanup_old_events())
yield
# Shutdown
+132
View File
@@ -0,0 +1,132 @@
"""Ollama vision LLM client for frigate-notify v2."""
import base64
import logging
import aiohttp
logger = logging.getLogger(__name__)
class OllamaClient:
"""Client for Ollama vision models (e.g., llava)."""
def __init__(self, base_url: str, model: str):
self.base_url = base_url.rstrip("/")
self.model = model
async def analyze_image(self, image_bytes: bytes, prompt: str) -> str | None:
"""Send an image to Ollama for analysis.
Args:
image_bytes: Raw image bytes (JPEG).
prompt: Text prompt for the vision model.
Returns:
Response text or None on failure.
"""
b64 = base64.b64encode(image_bytes).decode("utf-8")
payload = {
"model": self.model,
"prompt": prompt,
"images": [b64],
"stream": False,
"options": {"num_predict": 200},
}
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{self.base_url}/api/generate",
json=payload,
timeout=aiohttp.ClientTimeout(total=30),
) as resp:
if resp.status == 200:
data = await resp.json()
text = data.get("response", "").strip()
logger.info(
"Ollama analysis complete: %d chars", len(text)
)
return text if text else None
body = await resp.text()
logger.error(
"Ollama analyze_image failed %s: %s", resp.status, body
)
except Exception:
logger.exception("Ollama analyze_image request failed")
return None
async def compare_images(
self,
image_bytes: bytes,
reference_bytes: bytes,
object_name: str,
) -> bool | None:
"""Compare two images to determine if they show the same object/person.
Args:
image_bytes: The new event image bytes.
reference_bytes: The reference (known) image bytes.
object_name: Name of the known object for context.
Returns:
True if same, False if different, None on failure/inconclusive.
"""
b64_new = base64.b64encode(image_bytes).decode("utf-8")
b64_ref = base64.b64encode(reference_bytes).decode("utf-8")
prompt = (
f"Look at these two images. The first image is from a security camera. "
f"The second image is a reference photo of '{object_name}'. "
f"Are these the same person/vehicle/animal? "
f"Answer with ONLY 'yes' or 'no'."
)
payload = {
"model": self.model,
"prompt": prompt,
"images": [b64_new, b64_ref],
"stream": False,
"options": {"num_predict": 200},
}
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{self.base_url}/api/generate",
json=payload,
timeout=aiohttp.ClientTimeout(total=30),
) as resp:
if resp.status == 200:
data = await resp.json()
text = data.get("response", "").strip().lower()
logger.info("Ollama compare result: %s", text)
if "yes" in text.split()[:3]:
return True
if "no" in text.split()[:3]:
return False
logger.warning(
"Ollama compare inconclusive: %s", text
)
return None
body = await resp.text()
logger.error(
"Ollama compare_images failed %s: %s",
resp.status,
body,
)
except Exception:
logger.exception("Ollama compare_images request failed")
return None
async def is_available(self) -> bool:
"""Check if Ollama is reachable and responding.
Returns:
True if Ollama responds with HTTP 200, False otherwise.
"""
try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"{self.base_url}/api/tags",
timeout=aiohttp.ClientTimeout(total=3),
) as resp:
return resp.status == 200
except Exception:
return False
+232
View File
@@ -0,0 +1,232 @@
"""Recognition pipeline using CLIP embeddings with Ollama vision fallback."""
import base64
import logging
import struct
from typing import Optional
import aiohttp
import numpy as np
from config import config
from database import get_connection
from ollama_client import OllamaClient
logger = logging.getLogger(__name__)
def embedding_to_bytes(embedding: list[float]) -> bytes:
"""Pack a float32 embedding list into bytes for SQLite storage.
Args:
embedding: List of float values (typically 768-dimensional).
Returns:
Packed bytes (len(embedding) * 4 bytes).
"""
return struct.pack(f"{len(embedding)}f", *embedding)
def bytes_to_embedding(data: bytes) -> np.ndarray:
"""Unpack bytes back into a float32 numpy array.
Args:
data: Packed bytes from embedding_to_bytes().
Returns:
Numpy array of float32 values.
"""
count = len(data) // 4
values = struct.unpack(f"{count}f", data)
return np.array(values, dtype=np.float32)
async def get_clip_embedding(image_bytes: bytes) -> list[float] | None:
"""Get CLIP embedding for an image from the clip-service sidecar.
Args:
image_bytes: Raw image bytes (JPEG).
Returns:
List of float embedding values, or None on failure.
"""
b64 = base64.b64encode(image_bytes).decode("utf-8")
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{config.CLIP_SERVICE_URL}/embed",
json={"image": b64},
timeout=aiohttp.ClientTimeout(total=10),
) as resp:
if resp.status == 200:
data = await resp.json()
embedding = data.get("embedding")
if embedding:
logger.info(
"CLIP embedding received: %d dimensions",
len(embedding),
)
return embedding
logger.warning("CLIP response missing embedding field")
else:
body = await resp.text()
logger.error(
"CLIP embed failed %s: %s", resp.status, body
)
except Exception:
logger.exception("CLIP embedding request failed")
return None
async def find_best_match(
query_embedding: list[float],
) -> tuple[int | None, float, str | None]:
"""Find the best matching known object for a query embedding.
Loads all embeddings from the database, computes cosine similarity,
groups by known_object_id and takes the max score per object.
Args:
query_embedding: The query embedding to match against.
Returns:
Tuple of (best_object_id, best_score, best_name).
Returns (None, 0.0, None) if no embeddings exist.
"""
conn = get_connection()
cursor = conn.execute(
"""
SELECT e.id, e.known_object_id, e.embedding, ko.name
FROM embeddings e
JOIN known_objects ko ON e.known_object_id = ko.id
"""
)
rows = cursor.fetchall()
if not rows:
return (None, 0.0, None)
query_vec = np.array(query_embedding, dtype=np.float32)
query_norm = np.linalg.norm(query_vec)
if query_norm == 0:
return (None, 0.0, None)
query_vec = query_vec / query_norm
# Group scores by known_object_id, keeping max per object
best_per_object: dict[int, tuple[float, str]] = {}
for row in rows:
stored_vec = bytes_to_embedding(row["embedding"])
stored_norm = np.linalg.norm(stored_vec)
if stored_norm == 0:
continue
stored_vec = stored_vec / stored_norm
similarity = float(np.dot(query_vec, stored_vec))
obj_id = row["known_object_id"]
obj_name = row["name"]
if obj_id not in best_per_object or similarity > best_per_object[obj_id][0]:
best_per_object[obj_id] = (similarity, obj_name)
if not best_per_object:
return (None, 0.0, None)
best_id = max(best_per_object, key=lambda k: best_per_object[k][0])
best_score, best_name = best_per_object[best_id]
return (best_id, best_score, best_name)
async def recognize_event(
image_bytes: bytes,
ollama_client: Optional[OllamaClient] = None,
) -> tuple[str, int | None, float, str | None]:
"""Full recognition pipeline for a Frigate event image.
Steps:
1. Get CLIP embedding for the image.
2. Find the best matching known object.
3. High confidence (>= threshold_high): return "known".
4. Low confidence (< threshold_low): return "unknown".
5. Ambiguous range: use Ollama vision LLM to confirm/deny.
6. If LLM unavailable or inconclusive: return "ambiguous".
7. If no embedding could be obtained: return "unreviewed".
Args:
image_bytes: Raw image bytes (JPEG).
ollama_client: Optional OllamaClient for ambiguous case fallback.
Returns:
Tuple of (match_status, object_id, score, object_name).
match_status is one of: "known", "unknown", "ambiguous", "unreviewed".
"""
# Step 1: Get CLIP embedding
embedding = await get_clip_embedding(image_bytes)
if embedding is None:
logger.warning("Could not get CLIP embedding, returning unreviewed")
return ("unreviewed", None, 0.0, None)
# Step 2: Find best match
best_id, best_score, best_name = await find_best_match(embedding)
# No embeddings in DB at all
if best_id is None:
return ("unknown", None, 0.0, None)
high_threshold = config.RECOGNITION_THRESHOLD_HIGH
low_threshold = config.RECOGNITION_THRESHOLD_LOW
# Step 3: High confidence match
if best_score >= high_threshold:
logger.info(
"High confidence match: %s (%.3f)", best_name, best_score
)
return ("known", best_id, best_score, best_name)
# Step 4: Low confidence - clearly not a match
if best_score < low_threshold:
logger.info("Low confidence, no match (%.3f)", best_score)
return ("unknown", None, best_score, None)
# Step 5: Ambiguous range - try Ollama vision LLM
if ollama_client is not None:
logger.info(
"Ambiguous score %.3f for %s, trying Ollama comparison",
best_score,
best_name,
)
# Get a reference thumbnail from the embeddings table
conn = get_connection()
cursor = conn.execute(
"""
SELECT thumbnail FROM embeddings
WHERE known_object_id = ? AND thumbnail IS NOT NULL
LIMIT 1
""",
(best_id,),
)
ref_row = cursor.fetchone()
if ref_row and ref_row["thumbnail"] is not None:
result = await ollama_client.compare_images(
image_bytes, ref_row["thumbnail"], best_name
)
if result is True:
logger.info(
"Ollama confirmed match: %s (%.3f)",
best_name,
best_score,
)
return ("known", best_id, best_score, best_name)
elif result is False:
logger.info(
"Ollama denied match: %s (%.3f)", best_name, best_score
)
return ("unknown", None, best_score, None)
# result is None: inconclusive, fall through to ambiguous
# Step 6: LLM unavailable or inconclusive
logger.info(
"Ambiguous match: %s (%.3f)", best_name, best_score
)
return ("ambiguous", best_id, best_score, best_name)
+578
View File
@@ -0,0 +1,578 @@
// Frigate Alerts - Web UI
(function () {
'use strict';
let currentView = 'dashboard';
let currentTagEventId = null;
let refreshTimer = null;
let knownObjectsCache = [];
// --- Initialization ---
document.addEventListener('DOMContentLoaded', () => {
setupNavigation();
setupTagModal();
loadView('dashboard');
loadStats();
});
// --- Navigation ---
function setupNavigation() {
document.querySelectorAll('.sidebar-item').forEach(item => {
item.addEventListener('click', (e) => {
e.preventDefault();
const view = item.dataset.view;
loadView(view);
});
});
}
function setActiveNav(view) {
document.querySelectorAll('.sidebar-item').forEach(item => {
item.classList.toggle('active', item.dataset.view === view);
});
}
// --- View Router ---
function loadView(view) {
currentView = view;
setActiveNav(view);
clearAutoRefresh();
switch (view) {
case 'dashboard':
loadDashboard();
startAutoRefresh();
break;
case 'review':
loadReview();
startAutoRefresh();
break;
case 'known':
loadKnown();
break;
case 'settings':
loadSettings();
break;
}
}
// --- Auto Refresh ---
function startAutoRefresh() {
refreshTimer = setInterval(() => {
if (currentView === 'dashboard') {
loadDashboard();
} else if (currentView === 'review') {
loadReview();
}
loadStats();
}, 30000);
}
function clearAutoRefresh() {
if (refreshTimer) {
clearInterval(refreshTimer);
refreshTimer = null;
}
}
// --- Dashboard View ---
function loadDashboard() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading events</div>';
fetch('/api/events?limit=50')
.then(r => r.json())
.then(events => {
if (!events.length) {
content.innerHTML = `
<h1>Dashboard</h1>
<div class="empty-state">
<p>No events recorded yet.</p>
<p>Events will appear here when Frigate detects objects.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Dashboard</h1>
</div>
<div class="filters">
<select id="filter-camera" onchange="window._applyFilters()">
<option value="">All Cameras</option>
</select>
<select id="filter-label" onchange="window._applyFilters()">
<option value="">All Labels</option>
</select>
<select id="filter-status" onchange="window._applyFilters()">
<option value="">All Statuses</option>
<option value="known">Known</option>
<option value="unknown">Unknown</option>
<option value="ambiguous">Ambiguous</option>
<option value="unreviewed">Unreviewed</option>
</select>
</div>
<div id="events-list"></div>`;
populateFilterOptions(events);
renderEvents(events);
})
.catch(err => {
content.innerHTML = `<h1>Dashboard</h1><div class="empty-state"><p>Error loading events: ${err.message}</p></div>`;
});
}
function populateFilterOptions(events) {
const cameras = [...new Set(events.map(e => e.camera).filter(Boolean))].sort();
const labels = [...new Set(events.map(e => e.label).filter(Boolean))].sort();
const cameraSelect = document.getElementById('filter-camera');
const labelSelect = document.getElementById('filter-label');
if (cameraSelect) {
cameras.forEach(c => {
const opt = document.createElement('option');
opt.value = c;
opt.textContent = c;
cameraSelect.appendChild(opt);
});
}
if (labelSelect) {
labels.forEach(l => {
const opt = document.createElement('option');
opt.value = l;
opt.textContent = l;
labelSelect.appendChild(opt);
});
}
}
// Expose filter function globally for inline onchange handlers
window._applyFilters = function () {
const camera = document.getElementById('filter-camera')?.value || '';
const label = document.getElementById('filter-label')?.value || '';
const status = document.getElementById('filter-status')?.value || '';
let url = '/api/events?limit=50';
if (camera) url += `&camera=${encodeURIComponent(camera)}`;
if (label) url += `&label=${encodeURIComponent(label)}`;
if (status) url += `&status=${encodeURIComponent(status)}`;
fetch(url)
.then(r => r.json())
.then(events => renderEvents(events))
.catch(err => console.error('Filter error:', err));
};
function renderEvents(events) {
const container = document.getElementById('events-list');
if (!container) return;
if (!events.length) {
container.innerHTML = '<div class="empty-state"><p>No events match the current filters.</p></div>';
return;
}
container.innerHTML = events.map(event => `
<div class="event-card" data-event-id="${event.id}">
<img src="/api/events/${event.id}/snapshot"
onerror="this.style.display='none'"
alt="${event.label || 'snapshot'}">
<div class="event-info">
<div class="event-header">
<span>
<span class="event-camera">${escapeHtml(event.camera || 'unknown')}</span>
<span class="event-label"> &mdash; ${escapeHtml(event.label || 'unknown')}</span>
<span class="event-score">(${(event.score * 100).toFixed(0)}%)</span>
</span>
<span class="badge badge-${event.match_status || 'unreviewed'}">${event.match_status || 'unreviewed'}</span>
</div>
<div class="event-time">${formatTime(event.created_at)}</div>
${event.ai_description ? `<div class="event-description">${escapeHtml(event.ai_description)}</div>` : ''}
${event.matched_object_name ? `<div class="event-label">Identified as: <strong>${escapeHtml(event.matched_object_name)}</strong></div>` : ''}
<div class="event-actions">
<button class="btn btn-primary btn-sm" onclick="window._tagEvent('${event.id}')">Tag</button>
<button class="btn btn-sm" onclick="window._dismissEvent('${event.id}')">Dismiss</button>
</div>
</div>
</div>
`).join('');
}
// --- Review View ---
function loadReview() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading review queue</div>';
fetch('/api/events?limit=50&status=unreviewed')
.then(r => r.json())
.then(events => {
if (!events.length) {
content.innerHTML = `
<h1>Review Queue</h1>
<div class="empty-state">
<p>No events to review.</p>
<p>All events have been reviewed or dismissed.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Review Queue</h1>
<span>${events.length} event${events.length !== 1 ? 's' : ''} to review</span>
</div>
<div id="events-list"></div>`;
renderEvents(events);
})
.catch(err => {
content.innerHTML = `<h1>Review Queue</h1><div class="empty-state"><p>Error loading events: ${err.message}</p></div>`;
});
}
// --- Known Objects View ---
function loadKnown() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading known objects</div>';
fetch('/api/known-objects')
.then(r => r.json())
.then(objects => {
knownObjectsCache = objects;
if (!objects.length) {
content.innerHTML = `
<h1>Known Objects</h1>
<div class="empty-state">
<p>No known objects registered.</p>
<p>Tag events from the Dashboard to add known objects.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Known Objects</h1>
<span>${objects.length} object${objects.length !== 1 ? 's' : ''}</span>
</div>
<div class="known-grid">
${objects.map(obj => `
<div class="known-card" data-object-id="${obj.id}">
<h4>${escapeHtml(obj.name)}</h4>
<span class="type-badge">${escapeHtml(obj.type)}</span>
<div class="embedding-count">${obj.embedding_count || 0} embedding${(obj.embedding_count || 0) !== 1 ? 's' : ''}</div>
${obj.samples && obj.samples.length ? `
<div class="known-card-samples">
${obj.samples.map(s => `<img src="${s}" onerror="this.style.display='none'" alt="sample">`).join('')}
</div>
` : ''}
<div class="known-card-actions">
<button class="btn btn-danger btn-sm" onclick="window._deleteKnownObject('${obj.id}', '${escapeHtml(obj.name)}')">Delete</button>
</div>
</div>
`).join('')}
</div>`;
})
.catch(err => {
content.innerHTML = `<h1>Known Objects</h1><div class="empty-state"><p>Error loading objects: ${err.message}</p></div>`;
});
}
// --- Settings View ---
function loadSettings() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading settings</div>';
fetch('/api/settings')
.then(r => r.json())
.then(settings => {
content.innerHTML = `
<h1>Settings</h1>
<div class="settings-form">
<div class="settings-section">
<h3>Vision Analysis</h3>
<div class="form-group">
<label>Vision Backend</label>
<select id="setting-vision-backend">
<option value="haiku" ${settings.vision_backend === 'haiku' ? 'selected' : ''}>Claude Haiku</option>
<option value="ollama" ${settings.vision_backend === 'ollama' ? 'selected' : ''}>Ollama (Local)</option>
<option value="both" ${settings.vision_backend === 'both' ? 'selected' : ''}>Both (Haiku + Ollama)</option>
</select>
<div class="help-text">Which AI backend to use for snapshot analysis</div>
</div>
</div>
<div class="settings-section">
<h3>Recognition</h3>
<div class="form-group">
<label class="checkbox-label">
<input type="checkbox" id="setting-recognition-enabled" ${settings.recognition_enabled ? 'checked' : ''}>
Enable object recognition
</label>
<div class="help-text">Match detected objects against known objects database</div>
</div>
</div>
<div class="settings-section">
<h3>Data Retention</h3>
<div class="form-group">
<label>Event Retention (days)</label>
<input type="number" id="setting-retention-days" value="${settings.event_retention_days || 30}" min="1" max="365">
<div class="help-text">How long to keep event records and snapshots</div>
</div>
</div>
<div class="form-group">
<button class="btn btn-primary" onclick="window._saveSettings()">Save Settings</button>
</div>
</div>`;
})
.catch(err => {
content.innerHTML = `<h1>Settings</h1><div class="empty-state"><p>Error loading settings: ${err.message}</p></div>`;
});
}
window._saveSettings = function () {
const settings = {
vision_backend: document.getElementById('setting-vision-backend').value,
recognition_enabled: document.getElementById('setting-recognition-enabled').checked,
event_retention_days: parseInt(document.getElementById('setting-retention-days').value, 10)
};
fetch('/api/settings', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(settings)
})
.then(r => {
if (!r.ok) throw new Error('Failed to save settings');
return r.json();
})
.then(() => showToast('Settings saved', 'success'))
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Stats ---
function loadStats() {
fetch('/api/stats')
.then(r => r.json())
.then(stats => {
const el = document.getElementById('stats-summary');
if (!el) return;
el.innerHTML = `
<div class="stat-item">
<span>Total Events</span>
<span class="stat-value">${stats.total_events || 0}</span>
</div>
<div class="stat-item">
<span>Unreviewed</span>
<span class="stat-value">${stats.unreviewed || 0}</span>
</div>
<div class="stat-item">
<span>Known</span>
<span class="stat-value">${stats.known_matches || 0}</span>
</div>
<div class="stat-item">
<span>Unknown</span>
<span class="stat-value">${stats.unknown_events || 0}</span>
</div>
<div class="stat-item">
<span>Known Objects</span>
<span class="stat-value">${stats.known_objects_count || 0}</span>
</div>`;
})
.catch(() => {
// Stats are non-critical, silently fail
});
}
// --- Tag Modal ---
function setupTagModal() {
document.getElementById('tag-cancel').addEventListener('click', closeTagModal);
document.getElementById('tag-save').addEventListener('click', saveTag);
document.getElementById('tag-existing-object').addEventListener('change', (e) => {
const newFields = document.getElementById('tag-new-fields');
newFields.style.display = e.target.value ? 'none' : 'block';
});
// Close modal on backdrop click
document.getElementById('tag-modal').addEventListener('click', (e) => {
if (e.target === document.getElementById('tag-modal')) {
closeTagModal();
}
});
// Close modal on Escape key
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape') closeTagModal();
});
}
window._tagEvent = function (eventId) {
currentTagEventId = eventId;
// Show snapshot in modal
const snapshotContainer = document.getElementById('tag-modal-snapshot');
snapshotContainer.innerHTML = `<img src="/api/events/${eventId}/snapshot" onerror="this.style.display='none'" alt="snapshot">`;
// Load known objects for dropdown
fetch('/api/known-objects')
.then(r => r.json())
.then(objects => {
knownObjectsCache = objects;
const select = document.getElementById('tag-existing-object');
select.innerHTML = '<option value="">-- Create new --</option>';
objects.forEach(obj => {
const opt = document.createElement('option');
opt.value = obj.id;
opt.textContent = `${obj.name} (${obj.type})`;
select.appendChild(opt);
});
})
.catch(() => { });
// Reset form
document.getElementById('tag-name').value = '';
document.getElementById('tag-type').value = 'person';
document.getElementById('tag-new-fields').style.display = 'block';
document.getElementById('tag-existing-object').value = '';
document.getElementById('tag-modal').classList.remove('hidden');
};
function closeTagModal() {
document.getElementById('tag-modal').classList.add('hidden');
currentTagEventId = null;
}
function saveTag() {
if (!currentTagEventId) return;
const existingId = document.getElementById('tag-existing-object').value;
let body;
if (existingId) {
body = { known_object_id: existingId };
} else {
const name = document.getElementById('tag-name').value.trim();
const type = document.getElementById('tag-type').value;
if (!name) {
showToast('Please enter a name', 'error');
return;
}
body = { name: name, type: type };
}
fetch(`/api/events/${currentTagEventId}/tag`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
})
.then(r => {
if (!r.ok) throw new Error('Failed to tag event');
return r.json();
})
.then(() => {
showToast('Event tagged successfully', 'success');
closeTagModal();
loadView(currentView);
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
}
// --- Event Actions ---
window._dismissEvent = function (eventId) {
fetch(`/api/events/${eventId}/dismiss`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' }
})
.then(r => {
if (!r.ok) throw new Error('Failed to dismiss event');
return r.json();
})
.then(() => {
showToast('Event dismissed', 'success');
// Remove card from DOM for immediate feedback
const card = document.querySelector(`.event-card[data-event-id="${eventId}"]`);
if (card) {
card.style.opacity = '0';
card.style.transition = 'opacity 0.3s';
setTimeout(() => card.remove(), 300);
}
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Known Object Actions ---
window._deleteKnownObject = function (objectId, name) {
if (!confirm(`Delete known object "${name}"? This cannot be undone.`)) return;
fetch(`/api/known-objects/${objectId}`, {
method: 'DELETE'
})
.then(r => {
if (!r.ok) throw new Error('Failed to delete object');
return r.json();
})
.then(() => {
showToast(`Deleted "${name}"`, 'success');
loadKnown();
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Utilities ---
function escapeHtml(str) {
if (!str) return '';
const div = document.createElement('div');
div.textContent = str;
return div.innerHTML;
}
function formatTime(timestamp) {
if (!timestamp) return '';
const date = new Date(timestamp);
const now = new Date();
const diffMs = now - date;
const diffMins = Math.floor(diffMs / 60000);
const diffHours = Math.floor(diffMs / 3600000);
const diffDays = Math.floor(diffMs / 86400000);
if (diffMins < 1) return 'Just now';
if (diffMins < 60) return `${diffMins}m ago`;
if (diffHours < 24) return `${diffHours}h ago`;
if (diffDays < 7) return `${diffDays}d ago`;
return date.toLocaleDateString() + ' ' + date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
}
function showToast(message, type) {
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
document.body.appendChild(toast);
setTimeout(() => {
toast.style.opacity = '0';
toast.style.transition = 'opacity 0.3s';
setTimeout(() => toast.remove(), 300);
}, 3000);
}
})();
+61
View File
@@ -0,0 +1,61 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Frigate Alerts</title>
<link rel="stylesheet" href="/static/style.css">
</head>
<body>
<nav id="sidebar">
<div class="sidebar-header">
<h2>Frigate Alerts</h2>
</div>
<div class="sidebar-section">
<a href="#" class="sidebar-item active" data-view="dashboard">Dashboard</a>
<a href="#" class="sidebar-item" data-view="review">Review Queue</a>
<a href="#" class="sidebar-item" data-view="known">Known Objects</a>
<a href="#" class="sidebar-item" data-view="settings">Settings</a>
</div>
<div class="sidebar-section sidebar-stats">
<div id="stats-summary"></div>
</div>
</nav>
<main id="content">
<div id="main-content"></div>
</main>
<div id="tag-modal" class="modal hidden">
<div class="modal-content">
<h3>Tag as Known Object</h3>
<div id="tag-modal-snapshot"></div>
<div class="form-group">
<label>Existing object:</label>
<select id="tag-existing-object">
<option value="">-- Create new --</option>
</select>
</div>
<div id="tag-new-fields">
<div class="form-group">
<label>Name:</label>
<input type="text" id="tag-name" placeholder="e.g., Jeremy, Blue F-150, Max">
</div>
<div class="form-group">
<label>Type:</label>
<select id="tag-type">
<option value="person">Person</option>
<option value="vehicle">Vehicle</option>
<option value="pet">Pet</option>
</select>
</div>
</div>
<div class="modal-actions">
<button id="tag-save" class="btn btn-primary">Save</button>
<button id="tag-cancel" class="btn">Cancel</button>
</div>
</div>
</div>
<script src="/static/app.js"></script>
</body>
</html>
+534
View File
@@ -0,0 +1,534 @@
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: #1a1a2e;
color: #e0e0e0;
display: flex;
min-height: 100vh;
}
/* Sidebar */
#sidebar {
width: 220px;
background: #16213e;
padding: 20px 0;
position: fixed;
height: 100vh;
overflow-y: auto;
}
.sidebar-header {
padding: 0 20px 20px;
border-bottom: 1px solid #0f3460;
}
.sidebar-header h2 {
color: #00b4d8;
font-size: 1.2em;
}
.sidebar-section {
padding: 10px 0;
}
.sidebar-item {
display: block;
padding: 10px 20px;
color: #a0a0a0;
text-decoration: none;
transition: all 0.2s;
}
.sidebar-item:hover {
background: #0f3460;
color: #fff;
}
.sidebar-item.active {
background: #0f3460;
color: #00b4d8;
border-left: 3px solid #00b4d8;
}
.sidebar-stats {
padding: 20px;
font-size: 0.85em;
color: #888;
}
/* Main content */
#content {
margin-left: 220px;
flex: 1;
padding: 20px;
}
/* Page headers */
h1 {
margin-bottom: 20px;
}
.page-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
}
/* Filters */
.filters {
display: flex;
gap: 10px;
margin-bottom: 20px;
flex-wrap: wrap;
}
.filters select,
.filters input {
padding: 6px 10px;
background: #16213e;
border: 1px solid #0f3460;
color: #e0e0e0;
border-radius: 4px;
}
.filters select:focus,
.filters input:focus {
outline: none;
border-color: #00b4d8;
}
/* Event cards */
.event-card {
background: #16213e;
border-radius: 8px;
padding: 15px;
margin-bottom: 15px;
display: flex;
gap: 15px;
transition: border-color 0.2s;
border: 1px solid transparent;
}
.event-card:hover {
border-color: #0f3460;
}
.event-card img {
width: 160px;
height: 120px;
object-fit: cover;
border-radius: 4px;
background: #1a1a2e;
}
.event-info {
flex: 1;
}
.event-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
}
.event-camera {
font-weight: bold;
color: #00b4d8;
}
.event-label {
color: #a0a0a0;
}
.event-score {
color: #888;
font-size: 0.85em;
}
.event-description {
color: #ccc;
margin: 8px 0;
font-size: 0.9em;
line-height: 1.4;
}
.event-time {
color: #666;
font-size: 0.8em;
margin-bottom: 6px;
}
.event-actions {
margin-top: 10px;
display: flex;
gap: 8px;
}
/* Status badges */
.badge {
padding: 3px 10px;
border-radius: 12px;
font-size: 0.8em;
font-weight: bold;
}
.badge-known {
background: #2d6a4f;
color: #95d5b2;
}
.badge-unknown {
background: #9a4c1a;
color: #f4a261;
}
.badge-ambiguous {
background: #7c6f09;
color: #ffd166;
}
.badge-unreviewed {
background: #444;
color: #aaa;
}
/* Buttons */
.btn {
padding: 6px 14px;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 0.85em;
transition: all 0.2s;
color: #ccc;
background: #2a2a4a;
}
.btn:hover {
background: #3a3a5a;
color: #fff;
}
.btn-primary {
background: #0f3460;
color: #fff;
}
.btn-primary:hover {
background: #00b4d8;
}
.btn-danger {
background: #7c2929;
color: #fff;
}
.btn-danger:hover {
background: #c0392b;
}
.btn-sm {
padding: 4px 10px;
font-size: 0.8em;
}
/* Modal */
.modal {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.7);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
}
.modal.hidden {
display: none;
}
.modal-content {
background: #16213e;
border-radius: 8px;
padding: 25px;
width: 450px;
max-width: 90vw;
}
.modal-content h3 {
margin-bottom: 15px;
color: #00b4d8;
}
.modal-actions {
display: flex;
gap: 10px;
justify-content: flex-end;
margin-top: 15px;
}
#tag-modal-snapshot {
margin-bottom: 15px;
}
#tag-modal-snapshot img {
width: 100%;
max-height: 200px;
object-fit: cover;
border-radius: 4px;
}
/* Form elements */
.form-group {
margin-bottom: 12px;
}
.form-group label {
display: block;
margin-bottom: 4px;
color: #a0a0a0;
font-size: 0.9em;
}
.form-group input,
.form-group select {
width: 100%;
padding: 8px 10px;
background: #1a1a2e;
border: 1px solid #0f3460;
color: #e0e0e0;
border-radius: 4px;
}
.form-group input:focus,
.form-group select:focus {
outline: none;
border-color: #00b4d8;
}
.form-group input[type="checkbox"] {
width: auto;
margin-right: 8px;
}
.form-group .checkbox-label {
display: flex;
align-items: center;
cursor: pointer;
}
.form-group .help-text {
font-size: 0.8em;
color: #666;
margin-top: 4px;
}
/* Known objects grid */
.known-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
gap: 15px;
}
.known-card {
background: #16213e;
border-radius: 8px;
padding: 15px;
border: 1px solid transparent;
transition: border-color 0.2s;
}
.known-card:hover {
border-color: #0f3460;
}
.known-card h4 {
color: #00b4d8;
margin-bottom: 5px;
}
.known-card .type-badge {
font-size: 0.8em;
color: #a0a0a0;
text-transform: capitalize;
}
.known-card .embedding-count {
font-size: 0.8em;
color: #666;
margin-top: 6px;
}
.known-card .known-card-actions {
margin-top: 10px;
display: flex;
gap: 8px;
}
.known-card .known-card-samples {
display: flex;
gap: 5px;
margin-top: 10px;
overflow-x: auto;
}
.known-card .known-card-samples img {
width: 60px;
height: 60px;
object-fit: cover;
border-radius: 4px;
background: #1a1a2e;
}
/* Settings */
.settings-form {
max-width: 600px;
}
.settings-form .form-group {
margin-bottom: 15px;
}
.settings-section {
margin-bottom: 30px;
}
.settings-section h3 {
color: #00b4d8;
margin-bottom: 15px;
padding-bottom: 8px;
border-bottom: 1px solid #0f3460;
}
/* Stats in sidebar */
.stat-item {
display: flex;
justify-content: space-between;
margin-bottom: 8px;
}
.stat-value {
color: #00b4d8;
font-weight: bold;
}
/* Empty state */
.empty-state {
text-align: center;
color: #666;
padding: 60px 20px;
}
.empty-state p {
margin-top: 10px;
font-size: 0.9em;
}
/* Loading spinner */
.loading {
text-align: center;
padding: 40px;
color: #666;
}
.loading::after {
content: '';
display: inline-block;
width: 20px;
height: 20px;
border: 2px solid #0f3460;
border-top-color: #00b4d8;
border-radius: 50%;
animation: spin 0.8s linear infinite;
margin-left: 10px;
vertical-align: middle;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
/* Toast notifications */
.toast {
position: fixed;
bottom: 20px;
right: 20px;
padding: 12px 20px;
border-radius: 6px;
color: #fff;
font-size: 0.9em;
z-index: 2000;
animation: slideIn 0.3s ease-out;
}
.toast-success {
background: #2d6a4f;
}
.toast-error {
background: #7c2929;
}
@keyframes slideIn {
from {
transform: translateY(20px);
opacity: 0;
}
to {
transform: translateY(0);
opacity: 1;
}
}
/* Responsive */
@media (max-width: 768px) {
#sidebar {
width: 100%;
height: auto;
position: relative;
}
.sidebar-section {
display: flex;
flex-wrap: wrap;
}
.sidebar-item {
padding: 8px 15px;
}
.sidebar-stats {
display: none;
}
#content {
margin-left: 0;
}
body {
flex-direction: column;
}
.event-card {
flex-direction: column;
}
.event-card img {
width: 100%;
height: auto;
}
.known-grid {
grid-template-columns: 1fr;
}
}
+67
View File
@@ -1,7 +1,11 @@
import base64
import logging
from typing import Optional
import anthropic
from ollama_client import OllamaClient
logger = logging.getLogger(__name__)
PROMPT = (
@@ -40,3 +44,66 @@ def analyze_snapshot(client: anthropic.Anthropic, image_bytes: bytes) -> str | N
except Exception:
logger.exception("Claude Vision API failed")
return None
async def analyze_snapshot_ollama(
ollama_client: OllamaClient, image_bytes: bytes
) -> str | None:
"""Send a snapshot to Ollama vision model for description.
Args:
ollama_client: Configured OllamaClient instance.
image_bytes: Raw image bytes (JPEG).
Returns:
Description string or None on failure.
"""
return await ollama_client.analyze_image(image_bytes, PROMPT)
async def get_description(
image_bytes: bytes,
vision_backend: str,
claude_client: Optional[anthropic.Anthropic] = None,
ollama_client: Optional[OllamaClient] = None,
) -> str | None:
"""Get a scene description using the configured vision backend.
Args:
image_bytes: Raw image bytes (JPEG).
vision_backend: One of "haiku", "ollama", or "both".
"both" tries Ollama first, falls back to Haiku.
claude_client: Anthropic client (required for "haiku" and "both").
ollama_client: OllamaClient instance (required for "ollama" and "both").
Returns:
Description string or None if all backends fail.
"""
if vision_backend == "ollama":
if ollama_client is None:
logger.error("Ollama backend selected but no client provided")
return None
return await analyze_snapshot_ollama(ollama_client, image_bytes)
if vision_backend == "haiku":
if claude_client is None:
logger.error("Haiku backend selected but no client provided")
return None
return analyze_snapshot(claude_client, image_bytes)
if vision_backend == "both":
# Try Ollama first, fall back to Haiku
if ollama_client is not None:
result = await analyze_snapshot_ollama(ollama_client, image_bytes)
if result is not None:
return result
logger.info("Ollama failed, falling back to Haiku")
if claude_client is not None:
return analyze_snapshot(claude_client, image_bytes)
logger.error("Both backend selected but no clients provided")
return None
logger.error("Unknown vision backend: %s", vision_backend)
return None
+15
View File
@@ -0,0 +1,15 @@
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
RUN apt-get update && apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY app/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ .
EXPOSE 5200
CMD ["python", "-u", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5200"]
+194
View File
@@ -0,0 +1,194 @@
"""CLIP embedding service for frigate-notify.
Loads OpenAI ViT-L/14 CLIP model and exposes endpoints for generating
image embeddings and computing cosine similarity between embeddings.
Designed to run on a GPU machine (RTX 3090) as a sidecar service.
"""
import base64
import io
import logging
import time
from contextlib import asynccontextmanager
import numpy as np
import open_clip
import torch
from fastapi import FastAPI, HTTPException
from PIL import Image
from pydantic import BaseModel
logger = logging.getLogger("clip-service")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
# --- Pydantic models ---
class EmbedRequest(BaseModel):
image: str # base64-encoded image
class EmbedResponse(BaseModel):
embedding: list[float]
inference_ms: float
class SimilarityRequest(BaseModel):
embedding1: list[float]
embedding2: list[float]
class SimilarityResponse(BaseModel):
score: float
class BatchSimilarityRequest(BaseModel):
query_embedding: list[float]
candidate_embeddings: list[list[float]]
class BatchSimilarityResponse(BaseModel):
scores: list[float]
best_index: int
best_score: float
# --- Globals ---
model = None
preprocess = None
device = None
model_name = "ViT-L-14"
pretrained = "openai"
def load_model():
"""Load the CLIP model onto the best available device."""
global model, preprocess, device
if torch.cuda.is_available():
device = torch.device("cuda")
logger.info("CUDA available — using GPU: %s", torch.cuda.get_device_name(0))
else:
device = torch.device("cpu")
logger.warning("CUDA not available — falling back to CPU")
logger.info("Loading CLIP model %s (pretrained=%s)...", model_name, pretrained)
t0 = time.monotonic()
model, _, preprocess = open_clip.create_model_and_transforms(
model_name, pretrained=pretrained, device=device
)
model.eval()
elapsed = time.monotonic() - t0
logger.info("Model loaded in %.1fs on %s", elapsed, device)
@asynccontextmanager
async def lifespan(app: FastAPI):
load_model()
yield
app = FastAPI(title="CLIP Embedding Service", version="1.0.0", lifespan=lifespan)
# --- Utility functions ---
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
"""Compute cosine similarity between two vectors."""
dot = np.dot(a, b)
norm_a = np.linalg.norm(a)
norm_b = np.linalg.norm(b)
if norm_a == 0 or norm_b == 0:
return 0.0
return float(dot / (norm_a * norm_b))
# --- Endpoints ---
@app.get("/health")
async def health():
return {
"status": "healthy",
"device": str(device) if device else "not_loaded",
"model": f"{model_name}/{pretrained}",
}
@app.post("/embed", response_model=EmbedResponse)
async def embed(request: EmbedRequest):
if model is None or preprocess is None:
raise HTTPException(status_code=503, detail="Model not loaded")
# Decode base64 image
try:
image_bytes = base64.b64decode(request.image)
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
except Exception as e:
raise HTTPException(status_code=400, detail=f"Invalid image data: {e}")
# Preprocess and encode
try:
image_tensor = preprocess(image).unsqueeze(0).to(device)
t0 = time.monotonic()
with torch.no_grad():
embedding = model.encode_image(image_tensor)
inference_ms = (time.monotonic() - t0) * 1000
# Normalize the embedding
embedding = embedding / embedding.norm(dim=-1, keepdim=True)
embedding_list = embedding.squeeze(0).cpu().numpy().tolist()
except Exception as e:
raise HTTPException(status_code=500, detail=f"Inference failed: {e}")
return EmbedResponse(embedding=embedding_list, inference_ms=round(inference_ms, 2))
@app.post("/similarity", response_model=SimilarityResponse)
async def similarity(request: SimilarityRequest):
a = np.array(request.embedding1, dtype=np.float32)
b = np.array(request.embedding2, dtype=np.float32)
if a.shape != b.shape:
raise HTTPException(
status_code=400,
detail=f"Embedding dimensions must match: {a.shape} vs {b.shape}",
)
score = cosine_similarity(a, b)
return SimilarityResponse(score=round(score, 6))
@app.post("/batch-similarity", response_model=BatchSimilarityResponse)
async def batch_similarity(request: BatchSimilarityRequest):
query = np.array(request.query_embedding, dtype=np.float32)
if not request.candidate_embeddings:
raise HTTPException(status_code=400, detail="No candidate embeddings provided")
candidates = np.array(request.candidate_embeddings, dtype=np.float32)
if query.shape[0] != candidates.shape[1]:
raise HTTPException(
status_code=400,
detail=f"Dimension mismatch: query={query.shape[0]}, candidates={candidates.shape[1]}",
)
# Compute cosine similarity for all candidates at once
query_norm = query / (np.linalg.norm(query) + 1e-10)
candidate_norms = candidates / (
np.linalg.norm(candidates, axis=1, keepdims=True) + 1e-10
)
scores = candidate_norms @ query_norm
scores_list = [round(float(s), 6) for s in scores]
best_index = int(np.argmax(scores))
best_score = scores_list[best_index]
return BatchSimilarityResponse(
scores=scores_list, best_index=best_index, best_score=best_score
)
+7
View File
@@ -0,0 +1,7 @@
fastapi>=0.115.0
uvicorn[standard]>=0.34.0
open-clip-torch>=2.29.0
torch>=2.5.0
Pillow>=10.0.0
numpy>=1.26.0
python-multipart>=0.0.18
+27
View File
@@ -0,0 +1,27 @@
services:
clip-service:
build: .
container_name: clip-service
restart: unless-stopped
ports:
- "5200:5200"
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
mem_limit: 4G
memswap_limit: 4G
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:5200/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 120s
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "3"
+65
View File
@@ -0,0 +1,65 @@
"""Test configuration: mock heavy dependencies that aren't needed for unit tests."""
import sys
from unittest.mock import MagicMock
# Mock modules that are not installed in the test environment.
# The tests only exercise pure numpy math, not model inference.
for mod_name in [
"torch",
"open_clip",
"fastapi",
"fastapi.responses",
"pydantic",
"PIL",
"PIL.Image",
"uvicorn",
]:
if mod_name not in sys.modules:
sys.modules[mod_name] = MagicMock()
# Provide a real BaseModel stand-in so Pydantic models can be defined
# (they are referenced at import time in main.py).
import types
pydantic_mock = sys.modules["pydantic"]
class _FakeBaseModel:
def __init_subclass__(cls, **kwargs):
pass
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
pydantic_mock.BaseModel = _FakeBaseModel
# Provide a fake FastAPI and HTTPException so the module loads
fastapi_mock = sys.modules["fastapi"]
class _FakeFastAPI:
def __init__(self, **kwargs):
pass
def get(self, *a, **kw):
def decorator(fn):
return fn
return decorator
def post(self, *a, **kw):
def decorator(fn):
return fn
return decorator
class _FakeHTTPException(Exception):
def __init__(self, status_code=500, detail=""):
self.status_code = status_code
self.detail = detail
fastapi_mock.FastAPI = _FakeFastAPI
fastapi_mock.HTTPException = _FakeHTTPException
+130
View File
@@ -0,0 +1,130 @@
"""Tests for CLIP service math utilities.
These tests exercise the cosine similarity logic using pure numpy --
no GPU, no model loading, no network calls required.
"""
import numpy as np
import pytest
# Import the cosine_similarity function directly from the app module.
# We add the app directory to sys.path so the import works without
# installing the package.
import sys
import os
sys.path.insert(
0, os.path.join(os.path.dirname(__file__), "..", "app")
)
from main import cosine_similarity
class TestCosineSimilarity:
"""Test the cosine_similarity helper function."""
def test_identical_vectors(self):
"""Identical vectors should have cosine similarity of 1.0."""
v = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
score = cosine_similarity(v, v)
assert score == pytest.approx(1.0, abs=1e-6)
def test_identical_unit_vectors(self):
"""Identical unit vectors should also yield 1.0."""
v = np.array([0.0, 1.0, 0.0], dtype=np.float32)
score = cosine_similarity(v, v)
assert score == pytest.approx(1.0, abs=1e-6)
def test_orthogonal_vectors(self):
"""Orthogonal vectors should have cosine similarity of 0.0."""
a = np.array([1.0, 0.0, 0.0], dtype=np.float32)
b = np.array([0.0, 1.0, 0.0], dtype=np.float32)
score = cosine_similarity(a, b)
assert score == pytest.approx(0.0, abs=1e-6)
def test_opposite_vectors(self):
"""Opposite vectors should have cosine similarity of -1.0."""
a = np.array([1.0, 2.0, 3.0], dtype=np.float32)
b = np.array([-1.0, -2.0, -3.0], dtype=np.float32)
score = cosine_similarity(a, b)
assert score == pytest.approx(-1.0, abs=1e-6)
def test_zero_vector_returns_zero(self):
"""A zero vector should return 0.0 (guarded against division by zero)."""
a = np.array([0.0, 0.0, 0.0], dtype=np.float32)
b = np.array([1.0, 2.0, 3.0], dtype=np.float32)
score = cosine_similarity(a, b)
assert score == pytest.approx(0.0, abs=1e-6)
def test_high_dimensional(self):
"""Test with 768-dimensional vectors (CLIP ViT-L/14 output size)."""
rng = np.random.RandomState(42)
a = rng.randn(768).astype(np.float32)
# Similarity with itself should be 1.0
score = cosine_similarity(a, a)
assert score == pytest.approx(1.0, abs=1e-5)
class TestBatchSimilarity:
"""Test batch similarity logic (matching the endpoint's numpy math)."""
def _batch_similarity(
self, query: np.ndarray, candidates: np.ndarray
) -> tuple[list[float], int, float]:
"""Replicate the batch-similarity endpoint logic."""
query_norm = query / (np.linalg.norm(query) + 1e-10)
candidate_norms = candidates / (
np.linalg.norm(candidates, axis=1, keepdims=True) + 1e-10
)
scores = candidate_norms @ query_norm
scores_list = [float(s) for s in scores]
best_index = int(np.argmax(scores))
best_score = scores_list[best_index]
return scores_list, best_index, best_score
def test_best_match_index(self):
"""The candidate most similar to the query should be identified."""
query = np.array([1.0, 0.0, 0.0], dtype=np.float32)
candidates = np.array(
[
[0.0, 1.0, 0.0], # orthogonal
[0.9, 0.1, 0.0], # close match
[1.0, 0.0, 0.0], # exact match
[0.5, 0.5, 0.0], # partial match
],
dtype=np.float32,
)
scores, best_idx, best_score = self._batch_similarity(query, candidates)
assert best_idx == 2
assert best_score == pytest.approx(1.0, abs=1e-5)
assert len(scores) == 4
# Orthogonal candidate should score ~0
assert scores[0] == pytest.approx(0.0, abs=1e-5)
def test_single_candidate(self):
"""Batch with a single candidate should work correctly."""
query = np.array([1.0, 1.0], dtype=np.float32)
candidates = np.array([[1.0, 1.0]], dtype=np.float32)
scores, best_idx, best_score = self._batch_similarity(query, candidates)
assert best_idx == 0
assert best_score == pytest.approx(1.0, abs=1e-5)
def test_scores_ordering(self):
"""Scores should reflect relative similarity to the query."""
query = np.array([1.0, 0.0], dtype=np.float32)
candidates = np.array(
[
[0.5, 0.866], # ~60 degrees
[0.866, 0.5], # ~30 degrees
[1.0, 0.0], # 0 degrees (exact)
],
dtype=np.float32,
)
scores, best_idx, _ = self._batch_similarity(query, candidates)
# Exact match should be best
assert best_idx == 2
# 30 degrees should score higher than 60 degrees
assert scores[1] > scores[0]
+6
View File
@@ -15,6 +15,12 @@ services:
- MQTT_HOST=192.168.86.192
- MQTT_PORT=1883
- MATRIX_HOMESERVER=http://192.168.86.192:8008
- CLIP_SERVICE_URL=http://192.168.86.196:5200
- OLLAMA_URL=http://192.168.86.196:11434
- OLLAMA_MODEL=llava:13b
- DB_PATH=/data/frigate-notify.db
volumes:
- /mnt/docker-data/configs/frigate-notify:/data
mem_limit: 256M
memswap_limit: 256M
healthcheck:
+65
View File
@@ -0,0 +1,65 @@
"""Tests for event retention cleanup."""
import sqlite3
from database import init_db, set_db_path
def test_cleanup_deletes_old_events(db_path):
"""Old events beyond retention should be deleted."""
set_db_path(db_path)
init_db()
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
# Insert an old event (100 days ago)
conn.execute(
"""INSERT INTO events (frigate_event_id, camera, label, score, match_status, created_at)
VALUES ('old-1', 'cam', 'person', 0.9, 'unknown', datetime('now', '-100 days'))"""
)
# Insert a recent event
conn.execute(
"""INSERT INTO events (frigate_event_id, camera, label, score, match_status)
VALUES ('new-1', 'cam', 'person', 0.9, 'unknown')"""
)
conn.commit()
# Simulate cleanup with 90-day retention
result = conn.execute(
"DELETE FROM events WHERE created_at < datetime('now', '-90 days')"
)
conn.commit()
assert result.rowcount == 1
remaining = conn.execute("SELECT COUNT(*) as c FROM events").fetchone()["c"]
assert remaining == 1
conn.close()
def test_cleanup_preserves_recent_events(db_path):
"""Events within retention period should not be deleted."""
set_db_path(db_path)
init_db()
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
# Insert events from 30 and 60 days ago
conn.execute(
"""INSERT INTO events (frigate_event_id, camera, label, score, match_status, created_at)
VALUES ('mid-1', 'cam', 'person', 0.9, 'unknown', datetime('now', '-30 days'))"""
)
conn.execute(
"""INSERT INTO events (frigate_event_id, camera, label, score, match_status, created_at)
VALUES ('mid-2', 'cam', 'car', 0.8, 'known', datetime('now', '-60 days'))"""
)
conn.commit()
result = conn.execute(
"DELETE FROM events WHERE created_at < datetime('now', '-90 days')"
)
conn.commit()
assert result.rowcount == 0
remaining = conn.execute("SELECT COUNT(*) as c FROM events").fetchone()["c"]
assert remaining == 2
conn.close()
+82
View File
@@ -0,0 +1,82 @@
"""Tests for the recognition module — embedding serialization helpers."""
import numpy as np
import pytest
from recognition import bytes_to_embedding, embedding_to_bytes
class TestEmbeddingToBytes:
"""Tests for embedding_to_bytes()."""
def test_roundtrip_preserves_values(self):
"""Packing and unpacking should return the same values."""
original = [0.1, 0.2, 0.3, -0.5, 1.0, 0.0]
packed = embedding_to_bytes(original)
recovered = bytes_to_embedding(packed)
np.testing.assert_allclose(recovered, original, rtol=1e-6)
def test_correct_byte_length_768(self):
"""A 768-dim embedding should produce 768 * 4 = 3072 bytes."""
embedding = [0.0] * 768
packed = embedding_to_bytes(embedding)
assert len(packed) == 768 * 4
def test_correct_byte_length_arbitrary(self):
"""Byte length should always be len(embedding) * 4."""
for dim in [1, 10, 512, 1024]:
embedding = [1.0] * dim
packed = embedding_to_bytes(embedding)
assert len(packed) == dim * 4
def test_empty_embedding(self):
"""An empty embedding should produce 0 bytes."""
packed = embedding_to_bytes([])
assert len(packed) == 0
recovered = bytes_to_embedding(packed)
assert len(recovered) == 0
class TestBytesToEmbedding:
"""Tests for bytes_to_embedding()."""
def test_correct_shape(self):
"""Should return a 1-D array with the correct number of elements."""
embedding = [0.5] * 768
packed = embedding_to_bytes(embedding)
recovered = bytes_to_embedding(packed)
assert recovered.shape == (768,)
def test_correct_dtype(self):
"""Should return float32 numpy array."""
embedding = [0.1, 0.2, 0.3]
packed = embedding_to_bytes(embedding)
recovered = bytes_to_embedding(packed)
assert recovered.dtype == np.float32
def test_negative_values(self):
"""Negative values should survive roundtrip."""
original = [-1.0, -0.5, 0.0, 0.5, 1.0]
packed = embedding_to_bytes(original)
recovered = bytes_to_embedding(packed)
np.testing.assert_allclose(recovered, original, rtol=1e-6)
def test_large_values(self):
"""Large float values should survive roundtrip within float32 precision."""
original = [1e10, -1e10, 3.14159, 2.71828]
packed = embedding_to_bytes(original)
recovered = bytes_to_embedding(packed)
np.testing.assert_allclose(recovered, original, rtol=1e-5)
def test_realistic_clip_embedding(self):
"""Simulate a realistic CLIP embedding (768-dim, normalized)."""
rng = np.random.default_rng(42)
vec = rng.standard_normal(768).astype(np.float32)
vec = vec / np.linalg.norm(vec) # normalize
packed = embedding_to_bytes(vec.tolist())
recovered = bytes_to_embedding(packed)
assert recovered.shape == (768,)
assert recovered.dtype == np.float32
np.testing.assert_allclose(recovered, vec, rtol=1e-6)