"""VoiceCoach v2 — livekit-agents 1.5.""" import asyncio import json import logging import os import uuid from livekit import rtc from livekit.agents import ( Agent, AgentSession, AutoSubscribe, JobContext, RoomInputOptions, llm as lk_llm, ) from livekit.plugins import openai as lk_openai from livekit.plugins import silero import store from modes import DEFAULT_MODE, MODES, ModeConfig from tools import ALL_TOOLS from vocal import VocalAnalyzer logger = logging.getLogger(__name__) SPEACHES_URL = os.getenv("SPEACHES_URL", "http://192.168.86.27:8000") OPENCLAW_URL = os.getenv("OPENCLAW_URL", "http://voicecoach-openclaw:3100") OPENCLAW_TOKEN = os.getenv("OPENCLAW_TOKEN", "") WHISPER_MODEL = "deepdml/faster-whisper-large-v3-turbo-ct2" KOKORO_MODEL = "speaches-ai/Kokoro-82M-v1.0-ONNX-fp16" KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_nova") def _build_instructions(mode: ModeConfig, context: str = "") -> str: if context: return f"{mode.instructions}\n\n{context}" return mode.instructions class VoiceCoachAgent(Agent): def __init__(self, mode_name: str, context: str, session_id: str, room: rtc.Room): mode = MODES[mode_name] super().__init__( instructions=_build_instructions(mode, context), tools=ALL_TOOLS, ) self._mode_name = mode_name self._session_id = session_id self._room = room self._vocal = VocalAnalyzer() async def on_enter(self): await store.create_session(self._session_id, self._mode_name) self.session.generate_reply( instructions="Greet Jeremy warmly in one sentence. Be natural, not formal." ) async def on_exit(self): await store.end_session(self._session_id) async def on_user_turn_completed( self, turn_ctx: lk_llm.ChatContext, new_message: lk_llm.ChatMessage, ): content = new_message.text_content or "" mode = MODES[self._mode_name] # Inject vocal state tag into message if mode uses it if mode.vocal_feedback and content: vocal = self._vocal.current if vocal.energy > 0.02: tag = vocal.to_tag() # Update message content to include vocal context new_message.content = [f"{tag}\n{content}"] asyncio.create_task( store.add_message( self._session_id, "user", content, vocal_energy=self._vocal.current.energy, vocal_emotion=self._vocal.current.emotion, ) ) await self._push({ "type": "final_transcript", "text": content, "speaker": "user", }) def switch_mode(self, mode_name: str): if mode_name not in MODES: logger.warning("Unknown mode: %s", mode_name) return self._mode_name = mode_name self.update_instructions(_build_instructions(MODES[mode_name])) asyncio.create_task(self._push({"type": "mode_ack", "mode": mode_name})) logger.info("Mode → %s", mode_name) async def _push(self, payload: dict): try: data = json.dumps(payload).encode() await self._room.local_participant.publish_data(data, reliable=True) except Exception as e: logger.debug("Data push failed: %s", e) async def entrypoint(ctx: JobContext): await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) session_id = uuid.uuid4().hex[:8] context = await store.get_recent_context(limit=3) agent = VoiceCoachAgent( mode_name=DEFAULT_MODE, context=context, session_id=session_id, room=ctx.room, ) session = AgentSession( stt=lk_openai.STT( base_url=f"{SPEACHES_URL}/v1", model=WHISPER_MODEL, api_key="speaches", ), llm=lk_openai.LLM( base_url=f"{OPENCLAW_URL}/v1", model="openclaw/assistant", api_key=OPENCLAW_TOKEN, ), tts=lk_openai.TTS( base_url=f"{SPEACHES_URL}/v1", model=KOKORO_MODEL, voice=KOKORO_VOICE, api_key="speaches", ), vad=silero.VAD.load(), allow_interruptions=True, min_endpointing_delay=0.4, max_endpointing_delay=2.0, ) # Data channel → mode switching @ctx.room.on("data_received") def on_data(packet: rtc.DataPacket): try: msg = json.loads(packet.data.decode()) if msg.get("type") == "mode_change": agent.switch_mode(msg.get("mode", DEFAULT_MODE)) except Exception: pass # Session events → browser @session.on("user_input_transcribed") def on_transcribed(ev): asyncio.create_task(agent._push({ "type": "partial_transcript" if not ev.is_final else "final_transcript", "text": ev.transcript, "speaker": "user", })) @session.on("conversation_item_added") def on_item(ev): item = ev.item if hasattr(item, "role") and item.role == "assistant": text = item.text_content or "" if text: asyncio.create_task(agent._push({ "type": "agent_response", "text": text, "speaker": "assistant", })) asyncio.create_task( store.add_message(session_id, "assistant", text) ) await session.start( agent, room=ctx.room, room_input_options=RoomInputOptions(), )