feat: frigate-notify v2 (modular app, CLIP recognition) + clip-service

Splits the single main.py into config/database/models/routes/vision/recognition/
ollama_client/cleanup modules, adds a web UI, and introduces clip-service for
CLIP (ViT-L-14) embeddings.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-11 11:19:03 -05:00
parent 835a8d6644
commit dddc6734d0
20 changed files with 2600 additions and 88 deletions
@@ -9,6 +9,8 @@ RUN pip install --no-cache-dir -r requirements.txt
COPY app/ .
RUN mkdir -p /data
EXPOSE 5199
CMD ["python", "-u", "main.py"]
@@ -0,0 +1,35 @@
"""Automatic event cleanup based on retention settings."""
import asyncio
import logging
from database import get_connection
logger = logging.getLogger(__name__)
async def cleanup_old_events():
"""Delete events older than the configured retention period. Runs hourly."""
while True:
try:
conn = get_connection()
row = conn.execute(
"SELECT value FROM settings WHERE key='event_retention_days'"
).fetchone()
days = int(row["value"]) if row else 90
result = conn.execute(
"DELETE FROM events WHERE created_at < datetime('now', ? || ' days')",
(f"-{days}",),
)
conn.commit()
if result.rowcount > 0:
logger.info(
"Cleaned up %d events older than %d days",
result.rowcount,
days,
)
except Exception:
logger.exception("Cleanup failed")
await asyncio.sleep(3600)
@@ -0,0 +1,56 @@
"""Centralized configuration for frigate-notify v2."""
import os
class Config:
"""Application configuration read from environment variables."""
# MQTT
MQTT_HOST: str = os.environ.get("MQTT_HOST", "192.168.86.192")
MQTT_PORT: int = int(os.environ.get("MQTT_PORT", "1883"))
# Frigate
FRIGATE_URL: str = os.environ.get("FRIGATE_URL", "http://192.168.86.192:5000")
# Matrix
MATRIX_HOMESERVER: str = os.environ.get(
"MATRIX_HOMESERVER", "http://192.168.86.192:8008"
)
MATRIX_ACCESS_TOKEN: str = os.environ.get("MATRIX_ACCESS_TOKEN", "")
MATRIX_ROOM_ID: str = os.environ.get("MATRIX_ROOM_ID", "")
# Vision - CLIP service
CLIP_SERVICE_URL: str = os.environ.get(
"CLIP_SERVICE_URL", "http://192.168.86.150:5200"
)
# Vision - Ollama
OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://192.168.86.150:11434")
OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "llava:13b")
# Recognition thresholds
RECOGNITION_THRESHOLD_HIGH: float = float(
os.environ.get("RECOGNITION_THRESHOLD_HIGH", "0.85")
)
RECOGNITION_THRESHOLD_LOW: float = float(
os.environ.get("RECOGNITION_THRESHOLD_LOW", "0.60")
)
# Filtering
ALLOWED_LABELS: set = set(
os.environ.get("ALLOWED_LABELS", "person,car,dog,cat,bird").split(",")
)
MIN_SCORE: float = float(os.environ.get("MIN_SCORE", "0.7"))
RATE_LIMIT_SECONDS: int = int(os.environ.get("RATE_LIMIT_SECONDS", "60"))
DEDUP_WINDOW_SECONDS: int = int(os.environ.get("DEDUP_WINDOW_SECONDS", "30"))
# Server
SERVER_HOST: str = os.environ.get("SERVER_HOST", "0.0.0.0")
SERVER_PORT: int = int(os.environ.get("SERVER_PORT", "5199"))
# Database
DB_PATH: str = os.environ.get("DB_PATH", "/data/frigate-notify.db")
config = Config()
@@ -0,0 +1,109 @@
"""SQLite database schema and connection management for frigate-notify v2."""
import logging
import sqlite3
import threading
logger = logging.getLogger(__name__)
# Thread-local storage for database connections
_local = threading.local()
_db_path: str = "/data/frigate-notify.db"
def set_db_path(path: str) -> None:
"""Set the database path. Closes any existing connection first."""
global _db_path
close_connection()
_db_path = path
def get_connection() -> sqlite3.Connection:
"""Get a thread-local database connection with WAL mode and foreign keys enabled."""
if not hasattr(_local, "connection") or _local.connection is None:
conn = sqlite3.connect(_db_path)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
conn.row_factory = sqlite3.Row
_local.connection = conn
return _local.connection
def close_connection() -> None:
"""Close the thread-local database connection if one exists."""
if hasattr(_local, "connection") and _local.connection is not None:
_local.connection.close()
_local.connection = None
def init_db() -> None:
"""Create all tables and default settings."""
conn = get_connection()
cursor = conn.cursor()
cursor.executescript(
"""
CREATE TABLE IF NOT EXISTS known_objects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
type TEXT NOT NULL CHECK(type IN ('person', 'vehicle', 'pet')),
action TEXT NOT NULL DEFAULT 'quiet' CHECK(action IN ('quiet', 'suppress', 'alert')),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
known_object_id INTEGER NOT NULL,
embedding BLOB NOT NULL,
snapshot_url TEXT,
thumbnail BLOB,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (known_object_id) REFERENCES known_objects(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
frigate_event_id TEXT UNIQUE NOT NULL,
camera TEXT NOT NULL,
label TEXT NOT NULL,
score REAL NOT NULL,
snapshot BLOB,
embedding BLOB,
match_status TEXT NOT NULL DEFAULT 'unreviewed'
CHECK(match_status IN ('known', 'unknown', 'ambiguous', 'unreviewed')),
matched_object_id INTEGER,
match_confidence REAL,
ai_description TEXT,
notified INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (matched_object_id) REFERENCES known_objects(id) ON DELETE SET NULL
);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at);
CREATE INDEX IF NOT EXISTS idx_events_match_status ON events(match_status);
CREATE INDEX IF NOT EXISTS idx_events_camera ON events(camera);
CREATE INDEX IF NOT EXISTS idx_embeddings_known_object_id ON embeddings(known_object_id);
"""
)
# Insert default settings if they don't exist
defaults = {
"vision_backend": "haiku",
"recognition_enabled": "false",
"event_retention_days": "90",
}
for key, value in defaults.items():
cursor.execute(
"INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)",
(key, value),
)
conn.commit()
logger.info("Database initialized at %s", _db_path)
+190 -86
View File
@@ -1,3 +1,5 @@
"""frigate-notify v2 — FastAPI application with MQTT listener."""
import asyncio
import json
import logging
@@ -6,29 +8,28 @@ import signal
import sys
import time
from collections import defaultdict
from http.server import HTTPServer, BaseHTTPRequestHandler
from contextlib import asynccontextmanager
from pathlib import Path
from threading import Thread
import aiohttp
import anthropic
import paho.mqtt.client as mqtt
import uvicorn
from fastapi import FastAPI
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from vision import analyze_snapshot
from config import config
from database import close_connection, get_connection, init_db, set_db_path
from matrix_client import MatrixClient
# --- Configuration ---
MQTT_HOST = os.environ.get("MQTT_HOST", "192.168.86.192")
MQTT_PORT = int(os.environ.get("MQTT_PORT", "1883"))
FRIGATE_URL = os.environ.get("FRIGATE_URL", "http://192.168.86.192:5000")
MATRIX_HOMESERVER = os.environ.get("MATRIX_HOMESERVER", "http://192.168.86.192:8008")
MATRIX_ACCESS_TOKEN = os.environ["MATRIX_ACCESS_TOKEN"]
MATRIX_ROOM_ID = os.environ["MATRIX_ROOM_ID"]
HEALTH_PORT = int(os.environ.get("HEALTH_PORT", "5199"))
ALLOWED_LABELS = {"person", "car", "dog", "cat", "bird"}
MIN_SCORE = 0.7
RATE_LIMIT_SECONDS = 60
DEDUP_WINDOW_SECONDS = 30
from routes.events import router as events_router
from routes.known_objects import router as known_objects_router
from routes.settings import router as settings_router
from ollama_client import OllamaClient
from recognition import recognize_event, get_clip_embedding, embedding_to_bytes
from vision import get_description
from cleanup import cleanup_old_events
# --- Logging ---
logging.basicConfig(
@@ -44,46 +45,12 @@ last_notify_per_camera: defaultdict[str, float] = defaultdict(float)
events_processed = 0
last_event_time: str | None = None
mqtt_connected = False
# --- Health endpoint ---
class HealthHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def do_GET(self):
if self.path == "/health":
body = json.dumps(
{
"status": "ok",
"mqtt_connected": mqtt_connected,
"events_processed": events_processed,
"last_event": last_event_time,
}
)
encoded = body.encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
else:
self.send_response(404)
self.end_headers()
def log_message(self, format, *args):
pass # suppress access logs
def start_health_server():
server = HTTPServer(("0.0.0.0", HEALTH_PORT), HealthHandler)
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
logger.info("Health server started on port %d", HEALTH_PORT)
mqtt_client_instance: mqtt.Client | None = None
# --- Snapshot fetching ---
async def fetch_snapshot(event_id: str) -> bytes | None:
url = f"{FRIGATE_URL}/api/events/{event_id}/snapshot.jpg"
url = f"{config.FRIGATE_URL}/api/events/{event_id}/snapshot.jpg"
try:
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
@@ -103,7 +70,8 @@ async def fetch_snapshot(event_id: str) -> bytes | None:
# --- Event processing ---
async def process_event(
event_data: dict, claude: anthropic.Anthropic, matrix: MatrixClient
event_data: dict, claude: anthropic.Anthropic, matrix: MatrixClient,
ollama_client: OllamaClient = None,
):
global events_processed, last_event_time
@@ -119,27 +87,30 @@ async def process_event(
has_snapshot = after.get("has_snapshot", False)
# --- Filtering ---
if label not in ALLOWED_LABELS:
if label not in config.ALLOWED_LABELS:
return
if score < MIN_SCORE:
if score < config.MIN_SCORE:
return
if not has_snapshot:
return
# Dedup
now = time.time()
if event_id in recent_events and now - recent_events[event_id] < DEDUP_WINDOW_SECONDS:
if (
event_id in recent_events
and now - recent_events[event_id] < config.DEDUP_WINDOW_SECONDS
):
return
recent_events[event_id] = now
# Rate limit per camera
if now - last_notify_per_camera[camera] < RATE_LIMIT_SECONDS:
if now - last_notify_per_camera[camera] < config.RATE_LIMIT_SECONDS:
logger.info("Rate limited: %s on %s", label, camera)
return
last_notify_per_camera[camera] = now
# Cleanup old dedup entries
cutoff = now - DEDUP_WINDOW_SECONDS * 2
cutoff = now - config.DEDUP_WINDOW_SECONDS * 2
for eid in [k for k, v in recent_events.items() if v < cutoff]:
del recent_events[eid]
@@ -150,18 +121,84 @@ async def process_event(
# --- Fetch snapshot ---
snapshot = await fetch_snapshot(event_id)
# --- AI analysis ---
# --- Recognition ---
match_status = "unreviewed"
matched_id = None
match_conf = 0.0
matched_name = None
# Check if recognition is enabled
recognition_enabled = False
try:
conn = get_connection()
row = conn.execute("SELECT value FROM settings WHERE key='recognition_enabled'").fetchone()
if row:
recognition_enabled = row["value"].lower() == "true"
except Exception:
pass
if recognition_enabled and snapshot:
match_status, matched_id, match_conf, matched_name = await recognize_event(
snapshot, ollama_client
)
# Store embedding with event
embedding = await get_clip_embedding(snapshot)
if embedding:
embedding_blob = embedding_to_bytes(embedding)
try:
conn = get_connection()
conn.execute(
"UPDATE events SET embedding=?, match_status=?, matched_object_id=?, match_confidence=? WHERE frigate_event_id=?",
(embedding_blob, match_status, matched_id, match_conf, event_id),
)
conn.commit()
except Exception:
logger.exception("Failed to update event recognition")
# If known, check action - skip notification for quiet/suppress
if match_status == "known" and matched_id:
conn = get_connection()
obj = conn.execute("SELECT action FROM known_objects WHERE id=?", (matched_id,)).fetchone()
if obj and obj["action"] in ("quiet", "suppress"):
logger.info("Known object '%s' (%s), skipping notification", matched_name, obj["action"])
events_processed += 1
last_event_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
return
# --- AI description ---
vision_backend = "haiku"
try:
conn = get_connection()
row = conn.execute("SELECT value FROM settings WHERE key='vision_backend'").fetchone()
if row:
vision_backend = row["value"]
except Exception:
pass
description = None
if snapshot:
description = analyze_snapshot(claude, snapshot)
description = await get_description(snapshot, vision_backend, claude, ollama_client)
if not description:
description = f"{label.capitalize()} detected on {camera}."
# --- Store event in database ---
try:
conn = get_connection()
conn.execute(
"""INSERT OR IGNORE INTO events
(frigate_event_id, camera, label, score, snapshot, match_status, ai_description, notified)
VALUES (?, ?, ?, ?, ?, ?, ?, 1)""",
(event_id, camera, label, score, snapshot, match_status, description),
)
conn.commit()
except Exception:
logger.exception("Failed to store event %s in database", event_id)
# --- Format message ---
score_pct = int(score * 100)
ts = time.strftime("%I:%M:%S %p", time.localtime())
date_str = time.strftime("%b %d, %Y", time.localtime())
frigate_link = f"{FRIGATE_URL}/review?id={event_id}"
frigate_link = f"{config.FRIGATE_URL}/review?id={event_id}"
plain = (
f"{camera} - {label.capitalize()} - {score_pct}%\n"
@@ -195,7 +232,8 @@ def create_mqtt_client(
loop: asyncio.AbstractEventLoop,
claude: anthropic.Anthropic,
matrix: MatrixClient,
):
ollama_client: OllamaClient = None,
) -> mqtt.Client:
global mqtt_connected
def on_connect(client, userdata, flags, reason_code, properties):
@@ -213,7 +251,7 @@ def create_mqtt_client(
try:
payload = json.loads(msg.payload)
asyncio.run_coroutine_threadsafe(
process_event(payload, claude, matrix), loop
process_event(payload, claude, matrix, ollama_client), loop
)
except json.JSONDecodeError:
logger.warning("Invalid JSON on %s", msg.topic)
@@ -227,33 +265,99 @@ def create_mqtt_client(
return client
# --- Main ---
def main():
logger.info("Starting frigate-notify")
def start_mqtt(loop: asyncio.AbstractEventLoop) -> None:
"""Start MQTT client in a background thread."""
global mqtt_client_instance
claude = anthropic.Anthropic()
matrix = MatrixClient(MATRIX_HOMESERVER, MATRIX_ACCESS_TOKEN, MATRIX_ROOM_ID)
matrix = MatrixClient(
config.MATRIX_HOMESERVER,
config.MATRIX_ACCESS_TOKEN,
config.MATRIX_ROOM_ID,
)
ollama_client = OllamaClient(config.OLLAMA_URL, config.OLLAMA_MODEL)
loop = asyncio.new_event_loop()
loop_thread = Thread(target=loop.run_forever, daemon=True)
loop_thread.start()
mqtt_client_instance = create_mqtt_client(loop, claude, matrix, ollama_client)
mqtt_client_instance.connect(config.MQTT_HOST, config.MQTT_PORT)
start_health_server()
mqtt_client = create_mqtt_client(loop, claude, matrix)
mqtt_client.connect(MQTT_HOST, MQTT_PORT)
def shutdown(signum, frame):
logger.info("Shutting down...")
mqtt_client.disconnect()
loop.call_soon_threadsafe(loop.stop)
sys.exit(0)
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
mqtt_client.loop_forever()
thread = Thread(target=mqtt_client_instance.loop_forever, daemon=True)
thread.start()
logger.info("MQTT client started in background thread")
# --- FastAPI lifespan ---
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Startup and shutdown logic for the FastAPI application."""
logger.info("Starting frigate-notify v2")
# Initialize database
set_db_path(config.DB_PATH)
init_db()
# Start MQTT listener in background
loop = asyncio.get_event_loop()
start_mqtt(loop)
# Start periodic event cleanup
asyncio.create_task(cleanup_old_events())
yield
# Shutdown
logger.info("Shutting down frigate-notify v2")
if mqtt_client_instance:
mqtt_client_instance.disconnect()
close_connection()
# --- FastAPI app ---
app = FastAPI(
title="frigate-notify",
version="2.0.0",
lifespan=lifespan,
)
# Include API routers
app.include_router(events_router)
app.include_router(known_objects_router)
app.include_router(settings_router)
# Mount static files directory
static_dir = Path(__file__).parent / "static"
static_dir.mkdir(exist_ok=True)
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
@app.get("/health")
async def health():
"""Health check endpoint."""
return {
"status": "ok",
"version": "2.0.0",
"mqtt_connected": mqtt_connected,
"events_processed": events_processed,
"last_event": last_event_time,
}
@app.get("/")
async def index():
"""Serve the web UI index page."""
index_path = static_dir / "index.html"
if index_path.exists():
return FileResponse(str(index_path))
return JSONResponse(
{"message": "frigate-notify v2", "docs": "/docs"},
status_code=200,
)
# --- Entrypoint ---
if __name__ == "__main__":
main()
uvicorn.run(
"main:app",
host=config.SERVER_HOST,
port=config.SERVER_PORT,
log_level="info",
)
@@ -0,0 +1,95 @@
"""Pydantic models for frigate-notify v2 API."""
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field
# --- Known Objects ---
class KnownObjectCreate(BaseModel):
"""Request model for creating a known object."""
name: str = Field(..., min_length=1, max_length=255)
type: str = Field(..., pattern="^(person|vehicle|pet)$")
action: str = Field(default="quiet", pattern="^(quiet|suppress|alert)$")
class KnownObjectUpdate(BaseModel):
"""Request model for updating a known object."""
name: Optional[str] = Field(default=None, min_length=1, max_length=255)
type: Optional[str] = Field(default=None, pattern="^(person|vehicle|pet)$")
action: Optional[str] = Field(default=None, pattern="^(quiet|suppress|alert)$")
class KnownObjectResponse(BaseModel):
"""Response model for a known object."""
id: int
name: str
type: str
action: str
created_at: str
updated_at: str
# --- Events ---
class TagEventRequest(BaseModel):
"""Request model for tagging an event with a known object.
Either provide known_object_id to link to an existing object,
or provide name + type to create a new known object.
"""
known_object_id: Optional[int] = None
name: Optional[str] = Field(default=None, min_length=1, max_length=255)
type: Optional[str] = Field(default=None, pattern="^(person|vehicle|pet)$")
class EventResponse(BaseModel):
"""Response model for an event."""
id: int
frigate_event_id: str
camera: str
label: str
score: float
has_snapshot: bool = False
snapshot_base64: Optional[str] = None
match_status: str
matched_object_id: Optional[int] = None
matched_object_name: Optional[str] = None
match_confidence: Optional[float] = None
ai_description: Optional[str] = None
notified: bool
created_at: str
# --- Settings ---
class SettingsResponse(BaseModel):
"""Response model for application settings."""
vision_backend: str
recognition_enabled: bool
event_retention_days: int
# --- Stats ---
class StatsResponse(BaseModel):
"""Response model for application statistics."""
total_events: int = 0
known_matches: int = 0
unknown_events: int = 0
unreviewed: int = 0
known_objects_count: int = 0
events_today: int = 0
@@ -0,0 +1,132 @@
"""Ollama vision LLM client for frigate-notify v2."""
import base64
import logging
import aiohttp
logger = logging.getLogger(__name__)
class OllamaClient:
"""Client for Ollama vision models (e.g., llava)."""
def __init__(self, base_url: str, model: str):
self.base_url = base_url.rstrip("/")
self.model = model
async def analyze_image(self, image_bytes: bytes, prompt: str) -> str | None:
"""Send an image to Ollama for analysis.
Args:
image_bytes: Raw image bytes (JPEG).
prompt: Text prompt for the vision model.
Returns:
Response text or None on failure.
"""
b64 = base64.b64encode(image_bytes).decode("utf-8")
payload = {
"model": self.model,
"prompt": prompt,
"images": [b64],
"stream": False,
"options": {"num_predict": 200},
}
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{self.base_url}/api/generate",
json=payload,
timeout=aiohttp.ClientTimeout(total=30),
) as resp:
if resp.status == 200:
data = await resp.json()
text = data.get("response", "").strip()
logger.info(
"Ollama analysis complete: %d chars", len(text)
)
return text if text else None
body = await resp.text()
logger.error(
"Ollama analyze_image failed %s: %s", resp.status, body
)
except Exception:
logger.exception("Ollama analyze_image request failed")
return None
async def compare_images(
self,
image_bytes: bytes,
reference_bytes: bytes,
object_name: str,
) -> bool | None:
"""Compare two images to determine if they show the same object/person.
Args:
image_bytes: The new event image bytes.
reference_bytes: The reference (known) image bytes.
object_name: Name of the known object for context.
Returns:
True if same, False if different, None on failure/inconclusive.
"""
b64_new = base64.b64encode(image_bytes).decode("utf-8")
b64_ref = base64.b64encode(reference_bytes).decode("utf-8")
prompt = (
f"Look at these two images. The first image is from a security camera. "
f"The second image is a reference photo of '{object_name}'. "
f"Are these the same person/vehicle/animal? "
f"Answer with ONLY 'yes' or 'no'."
)
payload = {
"model": self.model,
"prompt": prompt,
"images": [b64_new, b64_ref],
"stream": False,
"options": {"num_predict": 200},
}
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{self.base_url}/api/generate",
json=payload,
timeout=aiohttp.ClientTimeout(total=30),
) as resp:
if resp.status == 200:
data = await resp.json()
text = data.get("response", "").strip().lower()
logger.info("Ollama compare result: %s", text)
if "yes" in text.split()[:3]:
return True
if "no" in text.split()[:3]:
return False
logger.warning(
"Ollama compare inconclusive: %s", text
)
return None
body = await resp.text()
logger.error(
"Ollama compare_images failed %s: %s",
resp.status,
body,
)
except Exception:
logger.exception("Ollama compare_images request failed")
return None
async def is_available(self) -> bool:
"""Check if Ollama is reachable and responding.
Returns:
True if Ollama responds with HTTP 200, False otherwise.
"""
try:
async with aiohttp.ClientSession() as session:
async with session.get(
f"{self.base_url}/api/tags",
timeout=aiohttp.ClientTimeout(total=3),
) as resp:
return resp.status == 200
except Exception:
return False
@@ -0,0 +1,232 @@
"""Recognition pipeline using CLIP embeddings with Ollama vision fallback."""
import base64
import logging
import struct
from typing import Optional
import aiohttp
import numpy as np
from config import config
from database import get_connection
from ollama_client import OllamaClient
logger = logging.getLogger(__name__)
def embedding_to_bytes(embedding: list[float]) -> bytes:
"""Pack a float32 embedding list into bytes for SQLite storage.
Args:
embedding: List of float values (typically 768-dimensional).
Returns:
Packed bytes (len(embedding) * 4 bytes).
"""
return struct.pack(f"{len(embedding)}f", *embedding)
def bytes_to_embedding(data: bytes) -> np.ndarray:
"""Unpack bytes back into a float32 numpy array.
Args:
data: Packed bytes from embedding_to_bytes().
Returns:
Numpy array of float32 values.
"""
count = len(data) // 4
values = struct.unpack(f"{count}f", data)
return np.array(values, dtype=np.float32)
async def get_clip_embedding(image_bytes: bytes) -> list[float] | None:
"""Get CLIP embedding for an image from the clip-service sidecar.
Args:
image_bytes: Raw image bytes (JPEG).
Returns:
List of float embedding values, or None on failure.
"""
b64 = base64.b64encode(image_bytes).decode("utf-8")
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{config.CLIP_SERVICE_URL}/embed",
json={"image": b64},
timeout=aiohttp.ClientTimeout(total=10),
) as resp:
if resp.status == 200:
data = await resp.json()
embedding = data.get("embedding")
if embedding:
logger.info(
"CLIP embedding received: %d dimensions",
len(embedding),
)
return embedding
logger.warning("CLIP response missing embedding field")
else:
body = await resp.text()
logger.error(
"CLIP embed failed %s: %s", resp.status, body
)
except Exception:
logger.exception("CLIP embedding request failed")
return None
async def find_best_match(
query_embedding: list[float],
) -> tuple[int | None, float, str | None]:
"""Find the best matching known object for a query embedding.
Loads all embeddings from the database, computes cosine similarity,
groups by known_object_id and takes the max score per object.
Args:
query_embedding: The query embedding to match against.
Returns:
Tuple of (best_object_id, best_score, best_name).
Returns (None, 0.0, None) if no embeddings exist.
"""
conn = get_connection()
cursor = conn.execute(
"""
SELECT e.id, e.known_object_id, e.embedding, ko.name
FROM embeddings e
JOIN known_objects ko ON e.known_object_id = ko.id
"""
)
rows = cursor.fetchall()
if not rows:
return (None, 0.0, None)
query_vec = np.array(query_embedding, dtype=np.float32)
query_norm = np.linalg.norm(query_vec)
if query_norm == 0:
return (None, 0.0, None)
query_vec = query_vec / query_norm
# Group scores by known_object_id, keeping max per object
best_per_object: dict[int, tuple[float, str]] = {}
for row in rows:
stored_vec = bytes_to_embedding(row["embedding"])
stored_norm = np.linalg.norm(stored_vec)
if stored_norm == 0:
continue
stored_vec = stored_vec / stored_norm
similarity = float(np.dot(query_vec, stored_vec))
obj_id = row["known_object_id"]
obj_name = row["name"]
if obj_id not in best_per_object or similarity > best_per_object[obj_id][0]:
best_per_object[obj_id] = (similarity, obj_name)
if not best_per_object:
return (None, 0.0, None)
best_id = max(best_per_object, key=lambda k: best_per_object[k][0])
best_score, best_name = best_per_object[best_id]
return (best_id, best_score, best_name)
async def recognize_event(
image_bytes: bytes,
ollama_client: Optional[OllamaClient] = None,
) -> tuple[str, int | None, float, str | None]:
"""Full recognition pipeline for a Frigate event image.
Steps:
1. Get CLIP embedding for the image.
2. Find the best matching known object.
3. High confidence (>= threshold_high): return "known".
4. Low confidence (< threshold_low): return "unknown".
5. Ambiguous range: use Ollama vision LLM to confirm/deny.
6. If LLM unavailable or inconclusive: return "ambiguous".
7. If no embedding could be obtained: return "unreviewed".
Args:
image_bytes: Raw image bytes (JPEG).
ollama_client: Optional OllamaClient for ambiguous case fallback.
Returns:
Tuple of (match_status, object_id, score, object_name).
match_status is one of: "known", "unknown", "ambiguous", "unreviewed".
"""
# Step 1: Get CLIP embedding
embedding = await get_clip_embedding(image_bytes)
if embedding is None:
logger.warning("Could not get CLIP embedding, returning unreviewed")
return ("unreviewed", None, 0.0, None)
# Step 2: Find best match
best_id, best_score, best_name = await find_best_match(embedding)
# No embeddings in DB at all
if best_id is None:
return ("unknown", None, 0.0, None)
high_threshold = config.RECOGNITION_THRESHOLD_HIGH
low_threshold = config.RECOGNITION_THRESHOLD_LOW
# Step 3: High confidence match
if best_score >= high_threshold:
logger.info(
"High confidence match: %s (%.3f)", best_name, best_score
)
return ("known", best_id, best_score, best_name)
# Step 4: Low confidence - clearly not a match
if best_score < low_threshold:
logger.info("Low confidence, no match (%.3f)", best_score)
return ("unknown", None, best_score, None)
# Step 5: Ambiguous range - try Ollama vision LLM
if ollama_client is not None:
logger.info(
"Ambiguous score %.3f for %s, trying Ollama comparison",
best_score,
best_name,
)
# Get a reference thumbnail from the embeddings table
conn = get_connection()
cursor = conn.execute(
"""
SELECT thumbnail FROM embeddings
WHERE known_object_id = ? AND thumbnail IS NOT NULL
LIMIT 1
""",
(best_id,),
)
ref_row = cursor.fetchone()
if ref_row and ref_row["thumbnail"] is not None:
result = await ollama_client.compare_images(
image_bytes, ref_row["thumbnail"], best_name
)
if result is True:
logger.info(
"Ollama confirmed match: %s (%.3f)",
best_name,
best_score,
)
return ("known", best_id, best_score, best_name)
elif result is False:
logger.info(
"Ollama denied match: %s (%.3f)", best_name, best_score
)
return ("unknown", None, best_score, None)
# result is None: inconclusive, fall through to ambiguous
# Step 6: LLM unavailable or inconclusive
logger.info(
"Ambiguous match: %s (%.3f)", best_name, best_score
)
return ("ambiguous", best_id, best_score, best_name)
@@ -1,3 +1,7 @@
paho-mqtt==2.1.0
anthropic>=0.40.0
aiohttp>=3.9.0
fastapi>=0.115.0
uvicorn[standard]>=0.34.0
python-multipart>=0.0.18
numpy>=1.26.0
@@ -0,0 +1,226 @@
"""Event API endpoints for frigate-notify v2."""
import base64
import logging
from typing import Optional
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import Response
from database import get_connection
from models import EventResponse, TagEventRequest
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/events", tags=["events"])
@router.get("", response_model=list[EventResponse])
async def list_events(
camera: Optional[str] = Query(default=None),
label: Optional[str] = Query(default=None),
status: Optional[str] = Query(default=None),
limit: int = Query(default=50, ge=1, le=500),
offset: int = Query(default=0, ge=0),
):
"""List events with optional filters and pagination.
Returns events without snapshot/embedding BLOBs but with has_snapshot bool.
Joins to known_objects for matched_object_name.
"""
conn = get_connection()
query = """
SELECT
e.id, e.frigate_event_id, e.camera, e.label, e.score,
CASE WHEN e.snapshot IS NOT NULL THEN 1 ELSE 0 END AS has_snapshot,
e.match_status, e.matched_object_id,
ko.name AS matched_object_name,
e.match_confidence, e.ai_description, e.notified, e.created_at
FROM events e
LEFT JOIN known_objects ko ON e.matched_object_id = ko.id
"""
conditions = []
params: list = []
if camera:
conditions.append("e.camera = ?")
params.append(camera)
if label:
conditions.append("e.label = ?")
params.append(label)
if status:
conditions.append("e.match_status = ?")
params.append(status)
if conditions:
query += " WHERE " + " AND ".join(conditions)
query += " ORDER BY e.created_at DESC LIMIT ? OFFSET ?"
params.extend([limit, offset])
cursor = conn.execute(query, params)
rows = cursor.fetchall()
return [
EventResponse(
id=row["id"],
frigate_event_id=row["frigate_event_id"],
camera=row["camera"],
label=row["label"],
score=row["score"],
has_snapshot=bool(row["has_snapshot"]),
match_status=row["match_status"],
matched_object_id=row["matched_object_id"],
matched_object_name=row["matched_object_name"],
match_confidence=row["match_confidence"],
ai_description=row["ai_description"],
notified=bool(row["notified"]),
created_at=row["created_at"],
)
for row in rows
]
@router.get("/{event_id}", response_model=EventResponse)
async def get_event(event_id: int):
"""Get a single event detail. Includes base64-encoded snapshot if exists."""
conn = get_connection()
cursor = conn.execute(
"""
SELECT
e.id, e.frigate_event_id, e.camera, e.label, e.score,
e.snapshot,
CASE WHEN e.snapshot IS NOT NULL THEN 1 ELSE 0 END AS has_snapshot,
e.match_status, e.matched_object_id,
ko.name AS matched_object_name,
e.match_confidence, e.ai_description, e.notified, e.created_at
FROM events e
LEFT JOIN known_objects ko ON e.matched_object_id = ko.id
WHERE e.id = ?
""",
(event_id,),
)
row = cursor.fetchone()
if not row:
raise HTTPException(status_code=404, detail="Event not found")
snapshot_b64 = None
if row["snapshot"] is not None:
snapshot_b64 = base64.b64encode(row["snapshot"]).decode("ascii")
return EventResponse(
id=row["id"],
frigate_event_id=row["frigate_event_id"],
camera=row["camera"],
label=row["label"],
score=row["score"],
has_snapshot=bool(row["has_snapshot"]),
snapshot_base64=snapshot_b64,
match_status=row["match_status"],
matched_object_id=row["matched_object_id"],
matched_object_name=row["matched_object_name"],
match_confidence=row["match_confidence"],
ai_description=row["ai_description"],
notified=bool(row["notified"]),
created_at=row["created_at"],
)
@router.get("/{event_id}/snapshot")
async def get_event_snapshot(event_id: int):
"""Return raw JPEG snapshot for an event. 404 if no snapshot."""
conn = get_connection()
cursor = conn.execute(
"SELECT snapshot FROM events WHERE id = ?",
(event_id,),
)
row = cursor.fetchone()
if not row:
raise HTTPException(status_code=404, detail="Event not found")
if row["snapshot"] is None:
raise HTTPException(status_code=404, detail="No snapshot available")
return Response(content=row["snapshot"], media_type="image/jpeg")
@router.post("/{event_id}/tag")
async def tag_event(event_id: int, body: TagEventRequest):
"""Tag an event as a known object.
Either provide known_object_id to link to an existing object,
or provide name + type to create a new known object.
If the event has an embedding, store it in the embeddings table.
"""
conn = get_connection()
# Verify event exists
cursor = conn.execute(
"SELECT id, embedding FROM events WHERE id = ?",
(event_id,),
)
event_row = cursor.fetchone()
if not event_row:
raise HTTPException(status_code=404, detail="Event not found")
known_object_id = body.known_object_id
if known_object_id is not None:
# Verify the known object exists
cursor = conn.execute(
"SELECT id FROM known_objects WHERE id = ?",
(known_object_id,),
)
if not cursor.fetchone():
raise HTTPException(status_code=404, detail="Known object not found")
elif body.name and body.type:
# Create a new known object
cursor = conn.execute(
"INSERT INTO known_objects (name, type) VALUES (?, ?)",
(body.name, body.type),
)
known_object_id = cursor.lastrowid
conn.commit()
else:
raise HTTPException(
status_code=400,
detail="Must provide either known_object_id or both name and type",
)
# If event has an embedding, store it in the embeddings table
if event_row["embedding"] is not None:
conn.execute(
"INSERT INTO embeddings (known_object_id, embedding) VALUES (?, ?)",
(known_object_id, event_row["embedding"]),
)
# Update event
conn.execute(
"""UPDATE events
SET match_status = 'known',
matched_object_id = ?,
match_confidence = 1.0
WHERE id = ?""",
(known_object_id, event_id),
)
conn.commit()
return {"status": "ok", "matched_object_id": known_object_id}
@router.post("/{event_id}/dismiss")
async def dismiss_event(event_id: int):
"""Set event match_status to 'unknown'."""
conn = get_connection()
cursor = conn.execute("SELECT id FROM events WHERE id = ?", (event_id,))
if not cursor.fetchone():
raise HTTPException(status_code=404, detail="Event not found")
conn.execute(
"UPDATE events SET match_status = 'unknown' WHERE id = ?",
(event_id,),
)
conn.commit()
return {"status": "ok"}
@@ -0,0 +1,177 @@
"""Known objects API endpoints for frigate-notify v2."""
import logging
from typing import Optional
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import Response
from database import get_connection
from models import KnownObjectResponse, KnownObjectUpdate
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/known-objects", tags=["known-objects"])
@router.get("")
async def list_known_objects(
type: Optional[str] = Query(default=None, pattern="^(person|vehicle|pet)$"),
):
"""List all known objects with embedding_count. Optional type filter."""
conn = get_connection()
query = """
SELECT
ko.id, ko.name, ko.type, ko.action,
ko.created_at, ko.updated_at,
COUNT(e.id) AS embedding_count
FROM known_objects ko
LEFT JOIN embeddings e ON ko.id = e.known_object_id
"""
params: list = []
if type:
query += " WHERE ko.type = ?"
params.append(type)
query += " GROUP BY ko.id ORDER BY ko.name"
cursor = conn.execute(query, params)
rows = cursor.fetchall()
return [
{
"id": row["id"],
"name": row["name"],
"type": row["type"],
"action": row["action"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
"embedding_count": row["embedding_count"],
}
for row in rows
]
@router.get("/{object_id}")
async def get_known_object(object_id: int):
"""Get a single known object with list of its embeddings (id, created_at only)."""
conn = get_connection()
cursor = conn.execute(
"SELECT id, name, type, action, created_at, updated_at FROM known_objects WHERE id = ?",
(object_id,),
)
row = cursor.fetchone()
if not row:
raise HTTPException(status_code=404, detail="Known object not found")
# Get embeddings list
emb_cursor = conn.execute(
"SELECT id, created_at FROM embeddings WHERE known_object_id = ? ORDER BY created_at",
(object_id,),
)
embeddings = [
{"id": e["id"], "created_at": e["created_at"]} for e in emb_cursor.fetchall()
]
return {
"id": row["id"],
"name": row["name"],
"type": row["type"],
"action": row["action"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
"embeddings": embeddings,
}
@router.get("/{object_id}/thumbnails/{embedding_id}")
async def get_embedding_thumbnail(object_id: int, embedding_id: int):
"""Return raw JPEG thumbnail from embeddings table."""
conn = get_connection()
cursor = conn.execute(
"SELECT thumbnail FROM embeddings WHERE id = ? AND known_object_id = ?",
(embedding_id, object_id),
)
row = cursor.fetchone()
if not row:
raise HTTPException(status_code=404, detail="Embedding not found")
if row["thumbnail"] is None:
raise HTTPException(status_code=404, detail="No thumbnail available")
return Response(content=row["thumbnail"], media_type="image/jpeg")
@router.put("/{object_id}")
async def update_known_object(object_id: int, body: KnownObjectUpdate):
"""Update name/type/action for a known object. Sets updated_at."""
conn = get_connection()
cursor = conn.execute(
"SELECT id FROM known_objects WHERE id = ?",
(object_id,),
)
if not cursor.fetchone():
raise HTTPException(status_code=404, detail="Known object not found")
updates = []
params: list = []
if body.name is not None:
updates.append("name = ?")
params.append(body.name)
if body.type is not None:
updates.append("type = ?")
params.append(body.type)
if body.action is not None:
updates.append("action = ?")
params.append(body.action)
if not updates:
raise HTTPException(status_code=400, detail="No fields to update")
updates.append("updated_at = CURRENT_TIMESTAMP")
params.append(object_id)
conn.execute(
f"UPDATE known_objects SET {', '.join(updates)} WHERE id = ?",
params,
)
conn.commit()
# Return the updated object
cursor = conn.execute(
"SELECT id, name, type, action, created_at, updated_at FROM known_objects WHERE id = ?",
(object_id,),
)
row = cursor.fetchone()
return KnownObjectResponse(
id=row["id"],
name=row["name"],
type=row["type"],
action=row["action"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
@router.delete("/{object_id}")
async def delete_known_object(object_id: int):
"""Delete a known object. Embeddings cascade delete automatically."""
conn = get_connection()
cursor = conn.execute(
"SELECT id FROM known_objects WHERE id = ?",
(object_id,),
)
if not cursor.fetchone():
raise HTTPException(status_code=404, detail="Known object not found")
conn.execute("DELETE FROM known_objects WHERE id = ?", (object_id,))
conn.commit()
return {"status": "ok"}
@@ -0,0 +1,78 @@
"""Settings and stats API endpoints for frigate-notify v2."""
import logging
from datetime import date
from fastapi import APIRouter
from config import config
from database import get_connection
from models import StatsResponse
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["settings"])
@router.get("/settings")
async def get_settings():
"""Return all settings from DB plus runtime config values."""
conn = get_connection()
cursor = conn.execute("SELECT key, value FROM settings")
db_settings = {row["key"]: row["value"] for row in cursor.fetchall()}
# Merge with runtime config values
return {
**db_settings,
"recognition_threshold_high": config.RECOGNITION_THRESHOLD_HIGH,
"recognition_threshold_low": config.RECOGNITION_THRESHOLD_LOW,
"rate_limit_seconds": config.RATE_LIMIT_SECONDS,
"allowed_labels": list(config.ALLOWED_LABELS),
"min_score": config.MIN_SCORE,
}
@router.put("/settings")
async def update_settings(body: dict):
"""Accept dict body, upsert each key/value into settings table."""
conn = get_connection()
for key, value in body.items():
conn.execute(
"INSERT INTO settings (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, str(value)),
)
conn.commit()
return {"status": "ok"}
@router.get("/stats", response_model=StatsResponse)
async def get_stats():
"""Return event and object counts."""
conn = get_connection()
total = conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]
known = conn.execute(
"SELECT COUNT(*) FROM events WHERE match_status = 'known'"
).fetchone()[0]
unknown = conn.execute(
"SELECT COUNT(*) FROM events WHERE match_status = 'unknown'"
).fetchone()[0]
unreviewed = conn.execute(
"SELECT COUNT(*) FROM events WHERE match_status = 'unreviewed'"
).fetchone()[0]
objects_count = conn.execute("SELECT COUNT(*) FROM known_objects").fetchone()[0]
today = date.today().isoformat()
events_today = conn.execute(
"SELECT COUNT(*) FROM events WHERE DATE(created_at) = ?",
(today,),
).fetchone()[0]
return StatsResponse(
total_events=total,
known_matches=known,
unknown_events=unknown,
unreviewed=unreviewed,
known_objects_count=objects_count,
events_today=events_today,
)
@@ -0,0 +1,578 @@
// Frigate Alerts - Web UI
(function () {
'use strict';
let currentView = 'dashboard';
let currentTagEventId = null;
let refreshTimer = null;
let knownObjectsCache = [];
// --- Initialization ---
document.addEventListener('DOMContentLoaded', () => {
setupNavigation();
setupTagModal();
loadView('dashboard');
loadStats();
});
// --- Navigation ---
function setupNavigation() {
document.querySelectorAll('.sidebar-item').forEach(item => {
item.addEventListener('click', (e) => {
e.preventDefault();
const view = item.dataset.view;
loadView(view);
});
});
}
function setActiveNav(view) {
document.querySelectorAll('.sidebar-item').forEach(item => {
item.classList.toggle('active', item.dataset.view === view);
});
}
// --- View Router ---
function loadView(view) {
currentView = view;
setActiveNav(view);
clearAutoRefresh();
switch (view) {
case 'dashboard':
loadDashboard();
startAutoRefresh();
break;
case 'review':
loadReview();
startAutoRefresh();
break;
case 'known':
loadKnown();
break;
case 'settings':
loadSettings();
break;
}
}
// --- Auto Refresh ---
function startAutoRefresh() {
refreshTimer = setInterval(() => {
if (currentView === 'dashboard') {
loadDashboard();
} else if (currentView === 'review') {
loadReview();
}
loadStats();
}, 30000);
}
function clearAutoRefresh() {
if (refreshTimer) {
clearInterval(refreshTimer);
refreshTimer = null;
}
}
// --- Dashboard View ---
function loadDashboard() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading events</div>';
fetch('/api/events?limit=50')
.then(r => r.json())
.then(events => {
if (!events.length) {
content.innerHTML = `
<h1>Dashboard</h1>
<div class="empty-state">
<p>No events recorded yet.</p>
<p>Events will appear here when Frigate detects objects.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Dashboard</h1>
</div>
<div class="filters">
<select id="filter-camera" onchange="window._applyFilters()">
<option value="">All Cameras</option>
</select>
<select id="filter-label" onchange="window._applyFilters()">
<option value="">All Labels</option>
</select>
<select id="filter-status" onchange="window._applyFilters()">
<option value="">All Statuses</option>
<option value="known">Known</option>
<option value="unknown">Unknown</option>
<option value="ambiguous">Ambiguous</option>
<option value="unreviewed">Unreviewed</option>
</select>
</div>
<div id="events-list"></div>`;
populateFilterOptions(events);
renderEvents(events);
})
.catch(err => {
content.innerHTML = `<h1>Dashboard</h1><div class="empty-state"><p>Error loading events: ${err.message}</p></div>`;
});
}
function populateFilterOptions(events) {
const cameras = [...new Set(events.map(e => e.camera).filter(Boolean))].sort();
const labels = [...new Set(events.map(e => e.label).filter(Boolean))].sort();
const cameraSelect = document.getElementById('filter-camera');
const labelSelect = document.getElementById('filter-label');
if (cameraSelect) {
cameras.forEach(c => {
const opt = document.createElement('option');
opt.value = c;
opt.textContent = c;
cameraSelect.appendChild(opt);
});
}
if (labelSelect) {
labels.forEach(l => {
const opt = document.createElement('option');
opt.value = l;
opt.textContent = l;
labelSelect.appendChild(opt);
});
}
}
// Expose filter function globally for inline onchange handlers
window._applyFilters = function () {
const camera = document.getElementById('filter-camera')?.value || '';
const label = document.getElementById('filter-label')?.value || '';
const status = document.getElementById('filter-status')?.value || '';
let url = '/api/events?limit=50';
if (camera) url += `&camera=${encodeURIComponent(camera)}`;
if (label) url += `&label=${encodeURIComponent(label)}`;
if (status) url += `&status=${encodeURIComponent(status)}`;
fetch(url)
.then(r => r.json())
.then(events => renderEvents(events))
.catch(err => console.error('Filter error:', err));
};
function renderEvents(events) {
const container = document.getElementById('events-list');
if (!container) return;
if (!events.length) {
container.innerHTML = '<div class="empty-state"><p>No events match the current filters.</p></div>';
return;
}
container.innerHTML = events.map(event => `
<div class="event-card" data-event-id="${event.id}">
<img src="/api/events/${event.id}/snapshot"
onerror="this.style.display='none'"
alt="${event.label || 'snapshot'}">
<div class="event-info">
<div class="event-header">
<span>
<span class="event-camera">${escapeHtml(event.camera || 'unknown')}</span>
<span class="event-label"> &mdash; ${escapeHtml(event.label || 'unknown')}</span>
<span class="event-score">(${(event.score * 100).toFixed(0)}%)</span>
</span>
<span class="badge badge-${event.match_status || 'unreviewed'}">${event.match_status || 'unreviewed'}</span>
</div>
<div class="event-time">${formatTime(event.created_at)}</div>
${event.ai_description ? `<div class="event-description">${escapeHtml(event.ai_description)}</div>` : ''}
${event.matched_object_name ? `<div class="event-label">Identified as: <strong>${escapeHtml(event.matched_object_name)}</strong></div>` : ''}
<div class="event-actions">
<button class="btn btn-primary btn-sm" onclick="window._tagEvent('${event.id}')">Tag</button>
<button class="btn btn-sm" onclick="window._dismissEvent('${event.id}')">Dismiss</button>
</div>
</div>
</div>
`).join('');
}
// --- Review View ---
function loadReview() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading review queue</div>';
fetch('/api/events?limit=50&status=unreviewed')
.then(r => r.json())
.then(events => {
if (!events.length) {
content.innerHTML = `
<h1>Review Queue</h1>
<div class="empty-state">
<p>No events to review.</p>
<p>All events have been reviewed or dismissed.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Review Queue</h1>
<span>${events.length} event${events.length !== 1 ? 's' : ''} to review</span>
</div>
<div id="events-list"></div>`;
renderEvents(events);
})
.catch(err => {
content.innerHTML = `<h1>Review Queue</h1><div class="empty-state"><p>Error loading events: ${err.message}</p></div>`;
});
}
// --- Known Objects View ---
function loadKnown() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading known objects</div>';
fetch('/api/known-objects')
.then(r => r.json())
.then(objects => {
knownObjectsCache = objects;
if (!objects.length) {
content.innerHTML = `
<h1>Known Objects</h1>
<div class="empty-state">
<p>No known objects registered.</p>
<p>Tag events from the Dashboard to add known objects.</p>
</div>`;
return;
}
content.innerHTML = `
<div class="page-header">
<h1>Known Objects</h1>
<span>${objects.length} object${objects.length !== 1 ? 's' : ''}</span>
</div>
<div class="known-grid">
${objects.map(obj => `
<div class="known-card" data-object-id="${obj.id}">
<h4>${escapeHtml(obj.name)}</h4>
<span class="type-badge">${escapeHtml(obj.type)}</span>
<div class="embedding-count">${obj.embedding_count || 0} embedding${(obj.embedding_count || 0) !== 1 ? 's' : ''}</div>
${obj.samples && obj.samples.length ? `
<div class="known-card-samples">
${obj.samples.map(s => `<img src="${s}" onerror="this.style.display='none'" alt="sample">`).join('')}
</div>
` : ''}
<div class="known-card-actions">
<button class="btn btn-danger btn-sm" onclick="window._deleteKnownObject('${obj.id}', '${escapeHtml(obj.name)}')">Delete</button>
</div>
</div>
`).join('')}
</div>`;
})
.catch(err => {
content.innerHTML = `<h1>Known Objects</h1><div class="empty-state"><p>Error loading objects: ${err.message}</p></div>`;
});
}
// --- Settings View ---
function loadSettings() {
const content = document.getElementById('main-content');
content.innerHTML = '<div class="loading">Loading settings</div>';
fetch('/api/settings')
.then(r => r.json())
.then(settings => {
content.innerHTML = `
<h1>Settings</h1>
<div class="settings-form">
<div class="settings-section">
<h3>Vision Analysis</h3>
<div class="form-group">
<label>Vision Backend</label>
<select id="setting-vision-backend">
<option value="haiku" ${settings.vision_backend === 'haiku' ? 'selected' : ''}>Claude Haiku</option>
<option value="ollama" ${settings.vision_backend === 'ollama' ? 'selected' : ''}>Ollama (Local)</option>
<option value="both" ${settings.vision_backend === 'both' ? 'selected' : ''}>Both (Haiku + Ollama)</option>
</select>
<div class="help-text">Which AI backend to use for snapshot analysis</div>
</div>
</div>
<div class="settings-section">
<h3>Recognition</h3>
<div class="form-group">
<label class="checkbox-label">
<input type="checkbox" id="setting-recognition-enabled" ${settings.recognition_enabled ? 'checked' : ''}>
Enable object recognition
</label>
<div class="help-text">Match detected objects against known objects database</div>
</div>
</div>
<div class="settings-section">
<h3>Data Retention</h3>
<div class="form-group">
<label>Event Retention (days)</label>
<input type="number" id="setting-retention-days" value="${settings.event_retention_days || 30}" min="1" max="365">
<div class="help-text">How long to keep event records and snapshots</div>
</div>
</div>
<div class="form-group">
<button class="btn btn-primary" onclick="window._saveSettings()">Save Settings</button>
</div>
</div>`;
})
.catch(err => {
content.innerHTML = `<h1>Settings</h1><div class="empty-state"><p>Error loading settings: ${err.message}</p></div>`;
});
}
window._saveSettings = function () {
const settings = {
vision_backend: document.getElementById('setting-vision-backend').value,
recognition_enabled: document.getElementById('setting-recognition-enabled').checked,
event_retention_days: parseInt(document.getElementById('setting-retention-days').value, 10)
};
fetch('/api/settings', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(settings)
})
.then(r => {
if (!r.ok) throw new Error('Failed to save settings');
return r.json();
})
.then(() => showToast('Settings saved', 'success'))
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Stats ---
function loadStats() {
fetch('/api/stats')
.then(r => r.json())
.then(stats => {
const el = document.getElementById('stats-summary');
if (!el) return;
el.innerHTML = `
<div class="stat-item">
<span>Total Events</span>
<span class="stat-value">${stats.total_events || 0}</span>
</div>
<div class="stat-item">
<span>Unreviewed</span>
<span class="stat-value">${stats.unreviewed || 0}</span>
</div>
<div class="stat-item">
<span>Known</span>
<span class="stat-value">${stats.known_matches || 0}</span>
</div>
<div class="stat-item">
<span>Unknown</span>
<span class="stat-value">${stats.unknown_events || 0}</span>
</div>
<div class="stat-item">
<span>Known Objects</span>
<span class="stat-value">${stats.known_objects_count || 0}</span>
</div>`;
})
.catch(() => {
// Stats are non-critical, silently fail
});
}
// --- Tag Modal ---
function setupTagModal() {
document.getElementById('tag-cancel').addEventListener('click', closeTagModal);
document.getElementById('tag-save').addEventListener('click', saveTag);
document.getElementById('tag-existing-object').addEventListener('change', (e) => {
const newFields = document.getElementById('tag-new-fields');
newFields.style.display = e.target.value ? 'none' : 'block';
});
// Close modal on backdrop click
document.getElementById('tag-modal').addEventListener('click', (e) => {
if (e.target === document.getElementById('tag-modal')) {
closeTagModal();
}
});
// Close modal on Escape key
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape') closeTagModal();
});
}
window._tagEvent = function (eventId) {
currentTagEventId = eventId;
// Show snapshot in modal
const snapshotContainer = document.getElementById('tag-modal-snapshot');
snapshotContainer.innerHTML = `<img src="/api/events/${eventId}/snapshot" onerror="this.style.display='none'" alt="snapshot">`;
// Load known objects for dropdown
fetch('/api/known-objects')
.then(r => r.json())
.then(objects => {
knownObjectsCache = objects;
const select = document.getElementById('tag-existing-object');
select.innerHTML = '<option value="">-- Create new --</option>';
objects.forEach(obj => {
const opt = document.createElement('option');
opt.value = obj.id;
opt.textContent = `${obj.name} (${obj.type})`;
select.appendChild(opt);
});
})
.catch(() => { });
// Reset form
document.getElementById('tag-name').value = '';
document.getElementById('tag-type').value = 'person';
document.getElementById('tag-new-fields').style.display = 'block';
document.getElementById('tag-existing-object').value = '';
document.getElementById('tag-modal').classList.remove('hidden');
};
function closeTagModal() {
document.getElementById('tag-modal').classList.add('hidden');
currentTagEventId = null;
}
function saveTag() {
if (!currentTagEventId) return;
const existingId = document.getElementById('tag-existing-object').value;
let body;
if (existingId) {
body = { known_object_id: existingId };
} else {
const name = document.getElementById('tag-name').value.trim();
const type = document.getElementById('tag-type').value;
if (!name) {
showToast('Please enter a name', 'error');
return;
}
body = { name: name, type: type };
}
fetch(`/api/events/${currentTagEventId}/tag`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
})
.then(r => {
if (!r.ok) throw new Error('Failed to tag event');
return r.json();
})
.then(() => {
showToast('Event tagged successfully', 'success');
closeTagModal();
loadView(currentView);
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
}
// --- Event Actions ---
window._dismissEvent = function (eventId) {
fetch(`/api/events/${eventId}/dismiss`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' }
})
.then(r => {
if (!r.ok) throw new Error('Failed to dismiss event');
return r.json();
})
.then(() => {
showToast('Event dismissed', 'success');
// Remove card from DOM for immediate feedback
const card = document.querySelector(`.event-card[data-event-id="${eventId}"]`);
if (card) {
card.style.opacity = '0';
card.style.transition = 'opacity 0.3s';
setTimeout(() => card.remove(), 300);
}
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Known Object Actions ---
window._deleteKnownObject = function (objectId, name) {
if (!confirm(`Delete known object "${name}"? This cannot be undone.`)) return;
fetch(`/api/known-objects/${objectId}`, {
method: 'DELETE'
})
.then(r => {
if (!r.ok) throw new Error('Failed to delete object');
return r.json();
})
.then(() => {
showToast(`Deleted "${name}"`, 'success');
loadKnown();
loadStats();
})
.catch(err => showToast('Error: ' + err.message, 'error'));
};
// --- Utilities ---
function escapeHtml(str) {
if (!str) return '';
const div = document.createElement('div');
div.textContent = str;
return div.innerHTML;
}
function formatTime(timestamp) {
if (!timestamp) return '';
const date = new Date(timestamp);
const now = new Date();
const diffMs = now - date;
const diffMins = Math.floor(diffMs / 60000);
const diffHours = Math.floor(diffMs / 3600000);
const diffDays = Math.floor(diffMs / 86400000);
if (diffMins < 1) return 'Just now';
if (diffMins < 60) return `${diffMins}m ago`;
if (diffHours < 24) return `${diffHours}h ago`;
if (diffDays < 7) return `${diffDays}d ago`;
return date.toLocaleDateString() + ' ' + date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
}
function showToast(message, type) {
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
document.body.appendChild(toast);
setTimeout(() => {
toast.style.opacity = '0';
toast.style.transition = 'opacity 0.3s';
setTimeout(() => toast.remove(), 300);
}, 3000);
}
})();
@@ -0,0 +1,61 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Frigate Alerts</title>
<link rel="stylesheet" href="/static/style.css">
</head>
<body>
<nav id="sidebar">
<div class="sidebar-header">
<h2>Frigate Alerts</h2>
</div>
<div class="sidebar-section">
<a href="#" class="sidebar-item active" data-view="dashboard">Dashboard</a>
<a href="#" class="sidebar-item" data-view="review">Review Queue</a>
<a href="#" class="sidebar-item" data-view="known">Known Objects</a>
<a href="#" class="sidebar-item" data-view="settings">Settings</a>
</div>
<div class="sidebar-section sidebar-stats">
<div id="stats-summary"></div>
</div>
</nav>
<main id="content">
<div id="main-content"></div>
</main>
<div id="tag-modal" class="modal hidden">
<div class="modal-content">
<h3>Tag as Known Object</h3>
<div id="tag-modal-snapshot"></div>
<div class="form-group">
<label>Existing object:</label>
<select id="tag-existing-object">
<option value="">-- Create new --</option>
</select>
</div>
<div id="tag-new-fields">
<div class="form-group">
<label>Name:</label>
<input type="text" id="tag-name" placeholder="e.g., Jeremy, Blue F-150, Max">
</div>
<div class="form-group">
<label>Type:</label>
<select id="tag-type">
<option value="person">Person</option>
<option value="vehicle">Vehicle</option>
<option value="pet">Pet</option>
</select>
</div>
</div>
<div class="modal-actions">
<button id="tag-save" class="btn btn-primary">Save</button>
<button id="tag-cancel" class="btn">Cancel</button>
</div>
</div>
</div>
<script src="/static/app.js"></script>
</body>
</html>
@@ -0,0 +1,534 @@
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: #1a1a2e;
color: #e0e0e0;
display: flex;
min-height: 100vh;
}
/* Sidebar */
#sidebar {
width: 220px;
background: #16213e;
padding: 20px 0;
position: fixed;
height: 100vh;
overflow-y: auto;
}
.sidebar-header {
padding: 0 20px 20px;
border-bottom: 1px solid #0f3460;
}
.sidebar-header h2 {
color: #00b4d8;
font-size: 1.2em;
}
.sidebar-section {
padding: 10px 0;
}
.sidebar-item {
display: block;
padding: 10px 20px;
color: #a0a0a0;
text-decoration: none;
transition: all 0.2s;
}
.sidebar-item:hover {
background: #0f3460;
color: #fff;
}
.sidebar-item.active {
background: #0f3460;
color: #00b4d8;
border-left: 3px solid #00b4d8;
}
.sidebar-stats {
padding: 20px;
font-size: 0.85em;
color: #888;
}
/* Main content */
#content {
margin-left: 220px;
flex: 1;
padding: 20px;
}
/* Page headers */
h1 {
margin-bottom: 20px;
}
.page-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
}
/* Filters */
.filters {
display: flex;
gap: 10px;
margin-bottom: 20px;
flex-wrap: wrap;
}
.filters select,
.filters input {
padding: 6px 10px;
background: #16213e;
border: 1px solid #0f3460;
color: #e0e0e0;
border-radius: 4px;
}
.filters select:focus,
.filters input:focus {
outline: none;
border-color: #00b4d8;
}
/* Event cards */
.event-card {
background: #16213e;
border-radius: 8px;
padding: 15px;
margin-bottom: 15px;
display: flex;
gap: 15px;
transition: border-color 0.2s;
border: 1px solid transparent;
}
.event-card:hover {
border-color: #0f3460;
}
.event-card img {
width: 160px;
height: 120px;
object-fit: cover;
border-radius: 4px;
background: #1a1a2e;
}
.event-info {
flex: 1;
}
.event-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
}
.event-camera {
font-weight: bold;
color: #00b4d8;
}
.event-label {
color: #a0a0a0;
}
.event-score {
color: #888;
font-size: 0.85em;
}
.event-description {
color: #ccc;
margin: 8px 0;
font-size: 0.9em;
line-height: 1.4;
}
.event-time {
color: #666;
font-size: 0.8em;
margin-bottom: 6px;
}
.event-actions {
margin-top: 10px;
display: flex;
gap: 8px;
}
/* Status badges */
.badge {
padding: 3px 10px;
border-radius: 12px;
font-size: 0.8em;
font-weight: bold;
}
.badge-known {
background: #2d6a4f;
color: #95d5b2;
}
.badge-unknown {
background: #9a4c1a;
color: #f4a261;
}
.badge-ambiguous {
background: #7c6f09;
color: #ffd166;
}
.badge-unreviewed {
background: #444;
color: #aaa;
}
/* Buttons */
.btn {
padding: 6px 14px;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 0.85em;
transition: all 0.2s;
color: #ccc;
background: #2a2a4a;
}
.btn:hover {
background: #3a3a5a;
color: #fff;
}
.btn-primary {
background: #0f3460;
color: #fff;
}
.btn-primary:hover {
background: #00b4d8;
}
.btn-danger {
background: #7c2929;
color: #fff;
}
.btn-danger:hover {
background: #c0392b;
}
.btn-sm {
padding: 4px 10px;
font-size: 0.8em;
}
/* Modal */
.modal {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.7);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
}
.modal.hidden {
display: none;
}
.modal-content {
background: #16213e;
border-radius: 8px;
padding: 25px;
width: 450px;
max-width: 90vw;
}
.modal-content h3 {
margin-bottom: 15px;
color: #00b4d8;
}
.modal-actions {
display: flex;
gap: 10px;
justify-content: flex-end;
margin-top: 15px;
}
#tag-modal-snapshot {
margin-bottom: 15px;
}
#tag-modal-snapshot img {
width: 100%;
max-height: 200px;
object-fit: cover;
border-radius: 4px;
}
/* Form elements */
.form-group {
margin-bottom: 12px;
}
.form-group label {
display: block;
margin-bottom: 4px;
color: #a0a0a0;
font-size: 0.9em;
}
.form-group input,
.form-group select {
width: 100%;
padding: 8px 10px;
background: #1a1a2e;
border: 1px solid #0f3460;
color: #e0e0e0;
border-radius: 4px;
}
.form-group input:focus,
.form-group select:focus {
outline: none;
border-color: #00b4d8;
}
.form-group input[type="checkbox"] {
width: auto;
margin-right: 8px;
}
.form-group .checkbox-label {
display: flex;
align-items: center;
cursor: pointer;
}
.form-group .help-text {
font-size: 0.8em;
color: #666;
margin-top: 4px;
}
/* Known objects grid */
.known-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
gap: 15px;
}
.known-card {
background: #16213e;
border-radius: 8px;
padding: 15px;
border: 1px solid transparent;
transition: border-color 0.2s;
}
.known-card:hover {
border-color: #0f3460;
}
.known-card h4 {
color: #00b4d8;
margin-bottom: 5px;
}
.known-card .type-badge {
font-size: 0.8em;
color: #a0a0a0;
text-transform: capitalize;
}
.known-card .embedding-count {
font-size: 0.8em;
color: #666;
margin-top: 6px;
}
.known-card .known-card-actions {
margin-top: 10px;
display: flex;
gap: 8px;
}
.known-card .known-card-samples {
display: flex;
gap: 5px;
margin-top: 10px;
overflow-x: auto;
}
.known-card .known-card-samples img {
width: 60px;
height: 60px;
object-fit: cover;
border-radius: 4px;
background: #1a1a2e;
}
/* Settings */
.settings-form {
max-width: 600px;
}
.settings-form .form-group {
margin-bottom: 15px;
}
.settings-section {
margin-bottom: 30px;
}
.settings-section h3 {
color: #00b4d8;
margin-bottom: 15px;
padding-bottom: 8px;
border-bottom: 1px solid #0f3460;
}
/* Stats in sidebar */
.stat-item {
display: flex;
justify-content: space-between;
margin-bottom: 8px;
}
.stat-value {
color: #00b4d8;
font-weight: bold;
}
/* Empty state */
.empty-state {
text-align: center;
color: #666;
padding: 60px 20px;
}
.empty-state p {
margin-top: 10px;
font-size: 0.9em;
}
/* Loading spinner */
.loading {
text-align: center;
padding: 40px;
color: #666;
}
.loading::after {
content: '';
display: inline-block;
width: 20px;
height: 20px;
border: 2px solid #0f3460;
border-top-color: #00b4d8;
border-radius: 50%;
animation: spin 0.8s linear infinite;
margin-left: 10px;
vertical-align: middle;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
/* Toast notifications */
.toast {
position: fixed;
bottom: 20px;
right: 20px;
padding: 12px 20px;
border-radius: 6px;
color: #fff;
font-size: 0.9em;
z-index: 2000;
animation: slideIn 0.3s ease-out;
}
.toast-success {
background: #2d6a4f;
}
.toast-error {
background: #7c2929;
}
@keyframes slideIn {
from {
transform: translateY(20px);
opacity: 0;
}
to {
transform: translateY(0);
opacity: 1;
}
}
/* Responsive */
@media (max-width: 768px) {
#sidebar {
width: 100%;
height: auto;
position: relative;
}
.sidebar-section {
display: flex;
flex-wrap: wrap;
}
.sidebar-item {
padding: 8px 15px;
}
.sidebar-stats {
display: none;
}
#content {
margin-left: 0;
}
body {
flex-direction: column;
}
.event-card {
flex-direction: column;
}
.event-card img {
width: 100%;
height: auto;
}
.known-grid {
grid-template-columns: 1fr;
}
}
@@ -1,7 +1,11 @@
import base64
import logging
from typing import Optional
import anthropic
from ollama_client import OllamaClient
logger = logging.getLogger(__name__)
PROMPT = (
@@ -40,3 +44,66 @@ def analyze_snapshot(client: anthropic.Anthropic, image_bytes: bytes) -> str | N
except Exception:
logger.exception("Claude Vision API failed")
return None
async def analyze_snapshot_ollama(
ollama_client: OllamaClient, image_bytes: bytes
) -> str | None:
"""Send a snapshot to Ollama vision model for description.
Args:
ollama_client: Configured OllamaClient instance.
image_bytes: Raw image bytes (JPEG).
Returns:
Description string or None on failure.
"""
return await ollama_client.analyze_image(image_bytes, PROMPT)
async def get_description(
image_bytes: bytes,
vision_backend: str,
claude_client: Optional[anthropic.Anthropic] = None,
ollama_client: Optional[OllamaClient] = None,
) -> str | None:
"""Get a scene description using the configured vision backend.
Args:
image_bytes: Raw image bytes (JPEG).
vision_backend: One of "haiku", "ollama", or "both".
"both" tries Ollama first, falls back to Haiku.
claude_client: Anthropic client (required for "haiku" and "both").
ollama_client: OllamaClient instance (required for "ollama" and "both").
Returns:
Description string or None if all backends fail.
"""
if vision_backend == "ollama":
if ollama_client is None:
logger.error("Ollama backend selected but no client provided")
return None
return await analyze_snapshot_ollama(ollama_client, image_bytes)
if vision_backend == "haiku":
if claude_client is None:
logger.error("Haiku backend selected but no client provided")
return None
return analyze_snapshot(claude_client, image_bytes)
if vision_backend == "both":
# Try Ollama first, fall back to Haiku
if ollama_client is not None:
result = await analyze_snapshot_ollama(ollama_client, image_bytes)
if result is not None:
return result
logger.info("Ollama failed, falling back to Haiku")
if claude_client is not None:
return analyze_snapshot(claude_client, image_bytes)
logger.error("Both backend selected but no clients provided")
return None
logger.error("Unknown vision backend: %s", vision_backend)
return None
@@ -12,8 +12,8 @@ services:
- MQTT_HOST=192.168.86.192
- MQTT_PORT=1883
- MATRIX_HOMESERVER=http://192.168.86.192:8008
- CLIP_SERVICE_URL=http://192.168.86.196:5200
- OLLAMA_URL=http://192.168.86.196:11434
- CLIP_SERVICE_URL=http://localhost:5200
- OLLAMA_URL=http://192.168.86.150:11434
- OLLAMA_MODEL=llava:13b
- DB_PATH=/data/frigate-notify.db
volumes: