feat: VoiceCoach v2 — livekit-agents 1.5 personal voice assistant

Replaces pipecat 0.0.48 (6 manual patches, DuplicateIdentity reconnect storms) with livekit-agents 1.5 SDK. New features: - SvelteKit UI with live transcript, VAD ring, mode bar, vocal meter, history view - SQLite session store with WAL mode (cross-session memory) - Async SpeechBrain vocal/emotion analysis injected per turn - Homelab tools: SSH/docker, Prometheus alerts, Home Assistant, SearXNG search - OpenClaw LLM gateway preserved (qwen2.5:14b via Ollama) - 4 modes: coaching, vent, practice, briefing Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-31 18:05:30 -05:00
parent 55f05a88d0
commit b3bef75981
24 changed files with 1660 additions and 0 deletions
@@ -0,0 +1,185 @@
+"""VoiceCoach v2 — livekit-agents 1.5."""
+import asyncio
+import json
+import logging
+import os
+import uuid
+
+from livekit import rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    AutoSubscribe,
+    JobContext,
+    RoomInputOptions,
+    llm as lk_llm,
+)
+from livekit.plugins import openai as lk_openai
+from livekit.plugins import silero
+
+import store
+from modes import DEFAULT_MODE, MODES, ModeConfig
+from tools import ALL_TOOLS
+from vocal import VocalAnalyzer
+
+logger = logging.getLogger(__name__)
+
+SPEACHES_URL = os.getenv("SPEACHES_URL", "http://192.168.86.27:8000")
+OPENCLAW_URL = os.getenv("OPENCLAW_URL", "http://voicecoach-openclaw:3100")
+OPENCLAW_TOKEN = os.getenv("OPENCLAW_TOKEN", "")
+WHISPER_MODEL = "deepdml/faster-whisper-large-v3-turbo-ct2"
+KOKORO_MODEL = "speaches-ai/Kokoro-82M-v1.0-ONNX-fp16"
+KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_nova")
+
+
+def _build_instructions(mode: ModeConfig, context: str = "") -> str:
+    if context:
+        return f"{mode.instructions}\n\n{context}"
+    return mode.instructions
+
+
+class VoiceCoachAgent(Agent):
+    def __init__(self, mode_name: str, context: str, session_id: str, room: rtc.Room):
+        mode = MODES[mode_name]
+        super().__init__(
+            instructions=_build_instructions(mode, context),
+            tools=ALL_TOOLS,
+        )
+        self._mode_name = mode_name
+        self._session_id = session_id
+        self._room = room
+        self._vocal = VocalAnalyzer()
+
+    async def on_enter(self):
+        await store.create_session(self._session_id, self._mode_name)
+        self.session.generate_reply(
+            instructions="Greet Jeremy warmly in one sentence. Be natural, not formal."
+        )
+
+    async def on_exit(self):
+        await store.end_session(self._session_id)
+
+    async def on_user_turn_completed(
+        self,
+        turn_ctx: lk_llm.ChatContext,
+        new_message: lk_llm.ChatMessage,
+    ):
+        content = new_message.text_content or ""
+        mode = MODES[self._mode_name]
+
+        # Inject vocal state tag into message if mode uses it
+        if mode.vocal_feedback and content:
+            vocal = self._vocal.current
+            if vocal.energy > 0.02:
+                tag = vocal.to_tag()
+                # Update message content to include vocal context
+                new_message.content = [f"{tag}\n{content}"]
+
+        asyncio.create_task(
+            store.add_message(
+                self._session_id,
+                "user",
+                content,
+                vocal_energy=self._vocal.current.energy,
+                vocal_emotion=self._vocal.current.emotion,
+            )
+        )
+
+        await self._push({
+            "type": "final_transcript",
+            "text": content,
+            "speaker": "user",
+        })
+
+    def switch_mode(self, mode_name: str):
+        if mode_name not in MODES:
+            logger.warning("Unknown mode: %s", mode_name)
+            return
+        self._mode_name = mode_name
+        self.update_instructions(_build_instructions(MODES[mode_name]))
+        asyncio.create_task(self._push({"type": "mode_ack", "mode": mode_name}))
+        logger.info("Mode → %s", mode_name)
+
+    async def _push(self, payload: dict):
+        try:
+            data = json.dumps(payload).encode()
+            await self._room.local_participant.publish_data(data, reliable=True)
+        except Exception as e:
+            logger.debug("Data push failed: %s", e)
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+
+    session_id = uuid.uuid4().hex[:8]
+    context = await store.get_recent_context(limit=3)
+
+    agent = VoiceCoachAgent(
+        mode_name=DEFAULT_MODE,
+        context=context,
+        session_id=session_id,
+        room=ctx.room,
+    )
+
+    session = AgentSession(
+        stt=lk_openai.STT(
+            base_url=f"{SPEACHES_URL}/v1",
+            model=WHISPER_MODEL,
+            api_key="speaches",
+        ),
+        llm=lk_openai.LLM(
+            base_url=f"{OPENCLAW_URL}/v1",
+            model="openclaw/assistant",
+            api_key=OPENCLAW_TOKEN,
+        ),
+        tts=lk_openai.TTS(
+            base_url=f"{SPEACHES_URL}/v1",
+            model=KOKORO_MODEL,
+            voice=KOKORO_VOICE,
+            api_key="speaches",
+        ),
+        vad=silero.VAD.load(),
+        allow_interruptions=True,
+        min_endpointing_delay=0.4,
+        max_endpointing_delay=2.0,
+    )
+
+    # Data channel → mode switching
+    @ctx.room.on("data_received")
+    def on_data(packet: rtc.DataPacket):
+        try:
+            msg = json.loads(packet.data.decode())
+            if msg.get("type") == "mode_change":
+                agent.switch_mode(msg.get("mode", DEFAULT_MODE))
+        except Exception:
+            pass
+
+    # Session events → browser
+    @session.on("user_input_transcribed")
+    def on_transcribed(ev):
+        asyncio.create_task(agent._push({
+            "type": "partial_transcript" if not ev.is_final else "final_transcript",
+            "text": ev.transcript,
+            "speaker": "user",
+        }))
+
+    @session.on("conversation_item_added")
+    def on_item(ev):
+        item = ev.item
+        if hasattr(item, "role") and item.role == "assistant":
+            text = item.text_content or ""
+            if text:
+                asyncio.create_task(agent._push({
+                    "type": "agent_response",
+                    "text": text,
+                    "speaker": "assistant",
+                }))
+                asyncio.create_task(
+                    store.add_message(session_id, "assistant", text)
+                )
+
+    await session.start(
+        agent,
+        room=ctx.room,
+        room_input_options=RoomInputOptions(),
+    )