hobbs-iac/compose/hf-pdocker-01/voicecoach-v2/agent/agent.py

"""VoiceCoach v2 — livekit-agents 1.5."""
import asyncio
import json
import logging
import os
import uuid

from livekit import rtc
from livekit.agents import (
    Agent,
    AgentSession,
    AutoSubscribe,
    JobContext,
    RoomInputOptions,
    llm as lk_llm,
)
from livekit.plugins import openai as lk_openai
from livekit.plugins import silero

import store
from modes import DEFAULT_MODE, MODES, ModeConfig
from tools import ALL_TOOLS
from vocal import VocalAnalyzer

logger = logging.getLogger(__name__)

SPEACHES_URL = os.getenv("SPEACHES_URL", "http://192.168.86.27:8000")
OPENCLAW_URL = os.getenv("OPENCLAW_URL", "http://voicecoach-openclaw:3100")
OPENCLAW_TOKEN = os.getenv("OPENCLAW_TOKEN", "")
WHISPER_MODEL = "deepdml/faster-whisper-large-v3-turbo-ct2"
KOKORO_MODEL = "speaches-ai/Kokoro-82M-v1.0-ONNX-fp16"
KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_nova")


def _build_instructions(mode: ModeConfig, context: str = "") -> str:
    if context:
        return f"{mode.instructions}\n\n{context}"
    return mode.instructions


class VoiceCoachAgent(Agent):
    def __init__(self, mode_name: str, context: str, session_id: str, room: rtc.Room):
        mode = MODES[mode_name]
        super().__init__(
            instructions=_build_instructions(mode, context),
            tools=ALL_TOOLS,
        )
        self._mode_name = mode_name
        self._session_id = session_id
        self._room = room
        self._vocal = VocalAnalyzer()

    async def on_enter(self):
        await store.create_session(self._session_id, self._mode_name)
        self.session.generate_reply(
            instructions="Greet Jeremy warmly in one sentence. Be natural, not formal."
        )

    async def on_exit(self):
        await store.end_session(self._session_id)

    async def on_user_turn_completed(
        self,
        turn_ctx: lk_llm.ChatContext,
        new_message: lk_llm.ChatMessage,
    ):
        content = new_message.text_content or ""
        mode = MODES[self._mode_name]

        # Inject vocal state tag into message if mode uses it
        if mode.vocal_feedback and content:
            vocal = self._vocal.current
            if vocal.energy > 0.02:
                tag = vocal.to_tag()
                # Update message content to include vocal context
                new_message.content = [f"{tag}\n{content}"]

        asyncio.create_task(
            store.add_message(
                self._session_id,
                "user",
                content,
                vocal_energy=self._vocal.current.energy,
                vocal_emotion=self._vocal.current.emotion,
            )
        )

        await self._push({
            "type": "final_transcript",
            "text": content,
            "speaker": "user",
        })

    def switch_mode(self, mode_name: str):
        if mode_name not in MODES:
            logger.warning("Unknown mode: %s", mode_name)
            return
        self._mode_name = mode_name
        self.update_instructions(_build_instructions(MODES[mode_name]))
        asyncio.create_task(self._push({"type": "mode_ack", "mode": mode_name}))
        logger.info("Mode → %s", mode_name)

    async def _push(self, payload: dict):
        try:
            data = json.dumps(payload).encode()
            await self._room.local_participant.publish_data(data, reliable=True)
        except Exception as e:
            logger.debug("Data push failed: %s", e)


async def entrypoint(ctx: JobContext):
    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)

    session_id = uuid.uuid4().hex[:8]
    context = await store.get_recent_context(limit=3)

    agent = VoiceCoachAgent(
        mode_name=DEFAULT_MODE,
        context=context,
        session_id=session_id,
        room=ctx.room,
    )

    session = AgentSession(
        stt=lk_openai.STT(
            base_url=f"{SPEACHES_URL}/v1",
            model=WHISPER_MODEL,
            api_key="speaches",
        ),
        llm=lk_openai.LLM(
            base_url=f"{OPENCLAW_URL}/v1",
            model="openclaw/assistant",
            api_key=OPENCLAW_TOKEN,
        ),
        tts=lk_openai.TTS(
            base_url=f"{SPEACHES_URL}/v1",
            model=KOKORO_MODEL,
            voice=KOKORO_VOICE,
            api_key="speaches",
        ),
        vad=silero.VAD.load(),
        allow_interruptions=True,
        min_endpointing_delay=0.4,
        max_endpointing_delay=2.0,
    )

    # Data channel → mode switching
    @ctx.room.on("data_received")
    def on_data(packet: rtc.DataPacket):
        try:
            msg = json.loads(packet.data.decode())
            if msg.get("type") == "mode_change":
                agent.switch_mode(msg.get("mode", DEFAULT_MODE))
        except Exception:
            pass

    # Session events → browser
    @session.on("user_input_transcribed")
    def on_transcribed(ev):
        asyncio.create_task(agent._push({
            "type": "partial_transcript" if not ev.is_final else "final_transcript",
            "text": ev.transcript,
            "speaker": "user",
        }))

    @session.on("conversation_item_added")
    def on_item(ev):
        item = ev.item
        if hasattr(item, "role") and item.role == "assistant":
            text = item.text_content or ""
            if text:
                asyncio.create_task(agent._push({
                    "type": "agent_response",
                    "text": text,
                    "speaker": "assistant",
                }))
                asyncio.create_task(
                    store.add_message(session_id, "assistant", text)
                )

    await session.start(
        agent,
        room=ctx.room,
        room_input_options=RoomInputOptions(),
    )