diff --git a/docker-compose.yml b/docker-compose.yml index f4dff14..5010ced 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: - # LiveKit WebRTC Server (unchanged) + # LiveKit WebRTC Server livekit-server: image: livekit/livekit-server:latest container_name: voicecoach-livekit @@ -27,24 +27,27 @@ services: max-size: "50m" max-file: "3" - # Speaches: self-hosted Whisper STT + Kokoro TTS (v2 - replaces Piper + in-process Whisper) + # Speaches: GPU-accelerated Whisper STT + Kokoro TTS speaches: - image: ghcr.io/speaches-ai/speaches:latest-cpu - container_name: voicecoach-speaches + image: ghcr.io/speaches-ai/speaches:latest-cuda + container_name: voicecoach-speaches-gpu restart: unless-stopped environment: - - WHISPER__MODEL=Systran/faster-whisper-large-v3-turbo - - WHISPER__INFERENCE_DEVICE=cpu - - WHISPER__COMPUTE_TYPE=int8 - - TTS__MODEL=kokoro - - TTS__DEFAULT_VOICE=af_nova - - TTS__LANGUAGE=en-us + - WHISPER__INFERENCE_DEVICE=cuda + - WHISPER__COMPUTE_TYPE=float16 + - WHISPER__TTL=-1 volumes: - - /mnt/docker-data/configs/voicecoach/speaches-models:/root/.cache + - speaches-models:/home/ubuntu/.cache networks: - voicecoach - mem_limit: 3G - memswap_limit: 3G + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + mem_limit: 4G labels: - "com.centurylinklabs.watchtower.enable=false" healthcheck: @@ -52,29 +55,28 @@ services: interval: 30s timeout: 10s retries: 3 - start_period: 120s + start_period: 180s logging: driver: "json-file" options: max-size: "50m" max-file: "3" - # SpeechBrain Emotion Analysis (unchanged, internal only) + # SpeechBrain Emotion Analysis speechbrain-service: + image: voicecoach-speechbrain:latest build: context: ./speechbrain-service - image: voicecoach-speechbrain:latest container_name: voicecoach-speechbrain restart: unless-stopped volumes: - - /mnt/docker-data/configs/voicecoach/speechbrain-models:/app/models - - /mnt/docker-data/configs/voicecoach/clips:/data/clips + - speechbrain-models:/app/models + - voicecoach-clips:/data/clips environment: - MODEL_DIR=/app/models networks: - voicecoach mem_limit: 3G - memswap_limit: 3G labels: - "com.centurylinklabs.watchtower.enable=false" logging: @@ -83,7 +85,7 @@ services: max-size: "50m" max-file: "3" - # OpenClaw VoiceCoach Agent (v2 - LLM gateway with memory/skills) + # OpenClaw VoiceCoach Agent (LLM gateway -> Claude) voicecoach-openclaw: image: margaret-openclaw:latest container_name: voicecoach-openclaw @@ -97,12 +99,11 @@ services: - OPENCLAW_STATE_DIR=/home/glyptic/.voicecoach - OPENCLAW_CONFIG_PATH=/home/glyptic/.voicecoach/openclaw.json volumes: - - /mnt/infrastructure/voicecoach:/home/glyptic/.voicecoach + - ./voicecoach:/home/glyptic/.voicecoach command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] networks: - voicecoach mem_limit: 2G - memswap_limit: 2G labels: - "com.centurylinklabs.watchtower.enable=false" logging: @@ -111,11 +112,11 @@ services: max-size: "50m" max-file: "3" - # Pipecat Voice Pipeline Agent (v2 - uses Speaches + OpenClaw gateway) + # Pipecat Voice Pipeline Agent pipecat-agent: + image: voicecoach-pipecat:latest build: context: ./pipecat-agent - image: voicecoach-pipecat:latest container_name: voicecoach-pipecat restart: unless-stopped env_file: @@ -125,22 +126,22 @@ services: - LIVEKIT_PUBLIC_URL=wss://voice.hobbs.farm - SPEACHES_URL=http://speaches:8000 - SPEECHBRAIN_URL=http://speechbrain-service:8001 - - OPENCLAW_URL=http://voicecoach-openclaw:3100 - - OPENCLAW_AGENT_ID=voicecoach - - WHISPER_MODEL=Systran/faster-whisper-large-v3-turbo + - LLM_BASE_URL=http://host.docker.internal:11434/v1 + - LLM_MODEL=llama3.1:8b + - WHISPER_MODEL=deepdml/faster-whisper-large-v3-turbo-ct2 - KOKORO_VOICE=af_nova - CLIPS_DIR=/data/clips/pending + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - - /mnt/docker-data/configs/voicecoach/clips:/data/clips + - voicecoach-clips:/data/clips depends_on: - livekit-server - speaches - speechbrain-service - - voicecoach-openclaw networks: - voicecoach mem_limit: 1G - memswap_limit: 1G labels: - "com.centurylinklabs.watchtower.enable=false" logging: @@ -149,17 +150,47 @@ services: max-size: "50m" max-file: "3" - # Web Client (unchanged) + # Token Server + Settings API (separate from pipeline so it never goes down) + token-server: + image: voicecoach-pipecat:latest + container_name: voicecoach-token-server + restart: unless-stopped + env_file: + - .env + environment: + - LIVEKIT_PUBLIC_URL=wss://voice.hobbs.farm + - SETTINGS_PATH=/data/settings.json + command: ["python", "token_server.py"] + volumes: + - voicecoach-settings:/data + networks: + - voicecoach + mem_limit: 256M + healthcheck: + test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8080/health').raise_for_status()"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + # Web Client voicecoach-web: + image: voicecoach-web:latest build: context: ./web-client - image: voicecoach-web:latest container_name: voicecoach-web restart: unless-stopped ports: - "8085:80" depends_on: - - pipecat-agent + - token-server networks: - voicecoach mem_limit: 128M @@ -174,3 +205,9 @@ services: networks: voicecoach: driver: bridge + +volumes: + speaches-models: + speechbrain-models: + voicecoach-clips: + voicecoach-settings: diff --git a/pipecat-agent/Dockerfile b/pipecat-agent/Dockerfile index a21f3ee..767f844 100755 --- a/pipecat-agent/Dockerfile +++ b/pipecat-agent/Dockerfile @@ -6,6 +6,11 @@ RUN apt-get update && apt-get install -y libsndfile1 ffmpeg && \ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt + +# Patch pipecat LiveKit transport bugs +COPY patch_livekit.py /tmp/patch_livekit.py +RUN python3 /tmp/patch_livekit.py + COPY . . HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ diff --git a/pipecat-agent/main.py b/pipecat-agent/main.py index 1c6d066..01f356f 100755 --- a/pipecat-agent/main.py +++ b/pipecat-agent/main.py @@ -1,18 +1,15 @@ """ -VoiceCoach pipeline entry point -- v2. -Runs token server + pipeline with unique identity per session. +VoiceCoach pipeline entry point. +Token server runs as a separate container — this file only runs the pipeline. +Pipeline exits on session end; Docker restarts the container for a fresh state. """ import asyncio import logging import os -import threading import uuid -from pathlib import Path +from datetime import timedelta import structlog -import uvicorn -from fastapi import FastAPI -from pydantic import BaseModel from livekit import api as livekit_api from pipeline import run_voice_coach_pipeline @@ -23,68 +20,23 @@ logger = structlog.get_logger() LIVEKIT_API_KEY = os.environ["LIVEKIT_API_KEY"] LIVEKIT_API_SECRET = os.environ["LIVEKIT_API_SECRET"] LIVEKIT_WS_URL = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880") -LIVEKIT_PUBLIC_URL = os.getenv("LIVEKIT_PUBLIC_URL", "wss://voice.hobbs.farm") -OPENCLAW_TOKEN = os.getenv("OPENCLAW_TOKEN", "") -# --- Token HTTP server --- - -token_app = FastAPI(title="VoiceCoach Token Service") - - -class TokenRequest(BaseModel): - identity: str = "jeremy" - - -class TokenResponse(BaseModel): - token: str - url: str - - -@token_app.get("/health") -async def health(): - return {"status": "ok", "version": "2.0"} - - -@token_app.post("/token") -async def get_token(req: TokenRequest): - token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) - token.with_grants(livekit_api.VideoGrants( - room_join=True, room="voicecoach", - can_publish=True, can_subscribe=True, - )) - token.identity = req.identity - return TokenResponse(token=token.to_jwt(), url=LIVEKIT_PUBLIC_URL) - - -def run_token_server(): - uvicorn.run(token_app, host="0.0.0.0", port=8080, log_level="info") - - -# --- Main --- async def main(): - server_thread = threading.Thread(target=run_token_server, daemon=True) - server_thread.start() - logger.info("Token server started on :8080") + agent_identity = f"voicecoach-agent-{uuid.uuid4().hex[:8]}" + token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) + token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) + token.identity = agent_identity + token.ttl = timedelta(hours=24) + livekit_token = token.to_jwt() - while True: - agent_identity = f"voicecoach-agent-{uuid.uuid4().hex[:8]}" - token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) - token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) - token.identity = agent_identity - livekit_token = token.to_jwt() - - logger.info("Starting VoiceCoach v2 pipeline", identity=agent_identity) - try: - await run_voice_coach_pipeline( - livekit_url=LIVEKIT_WS_URL, - livekit_token=livekit_token, - openclaw_token=OPENCLAW_TOKEN, - ) - except Exception as e: - logger.error("Pipeline error, restarting in 35s", error=str(e)) - - await asyncio.sleep(35) + logger.info("Starting VoiceCoach pipeline", identity=agent_identity) + await run_voice_coach_pipeline( + livekit_url=LIVEKIT_WS_URL, + livekit_token=livekit_token, + openclaw_token="", + ) + logger.info("Pipeline exited") if __name__ == "__main__": diff --git a/pipecat-agent/patch_livekit.py b/pipecat-agent/patch_livekit.py new file mode 100644 index 0000000..3be2641 --- /dev/null +++ b/pipecat-agent/patch_livekit.py @@ -0,0 +1,58 @@ +"""Patch pipecat LiveKit transport bugs at build time.""" +import pipecat.transports.services.livekit as lk_mod + +lk_path = lk_mod.__file__ +print(f"Patching {lk_path}") +content = open(lk_path).read() + +# FIX 1: Remove direct push_frame(pipecat_audio_frame) that bypasses VAD +old1 = "await self.push_frame(pipecat_audio_frame)" +if old1 in content: + content = content.replace(old1, "pass # PATCHED: removed VAD bypass") + print("FIX 1: Removed push_frame(pipecat_audio_frame) VAD bypass") + +# FIX 2: Wrap the on_connected call in connect() method with try/except +# The line is: " await self._callbacks.on_connected()" (12 spaces) +# We replace it with a try/except at the same indentation level +old2 = " await self._callbacks.on_connected()\n" +new2 = ( + " try:\n" + " await self._callbacks.on_connected()\n" + " except Exception:\n" + " pass # PATCHED: suppress duplicate callback\n" +) +if old2 in content: + content = content.replace(old2, new2) + print("FIX 2: Wrapped on_connected with try/except") + +open(lk_path, "w").write(content) + +# Verify syntax +import py_compile +try: + py_compile.compile(lk_path, doraise=True) + print("Syntax check: PASS") +except py_compile.PyCompileError as e: + print(f"Syntax check: FAIL - {e}") + # Show the problematic area + lines = open(lk_path).readlines() + for i in range(130, min(145, len(lines))): + print(f" {i+1}: {lines[i]}", end="") + +# FIX 3: Guard set_result on already-resolved Future in livekit SDK +try: + import livekit.rtc.room as room_mod + room_path = room_mod.__file__ + room_content = open(room_path).read() + if ("self._first_sid_future.set_result" in room_content and + ".done()" not in room_content.split("_first_sid_future")[1][:200]): + room_content = room_content.replace( + "self._first_sid_future.set_result", + "not self._first_sid_future.done() and self._first_sid_future.set_result" + ) + open(room_path, "w").write(room_content) + print(f"FIX 3: Patched {room_path}") +except Exception as e: + print(f"FIX 3: Skipped: {e}") + +print("All patches complete") diff --git a/pipecat-agent/pipeline.py b/pipecat-agent/pipeline.py index fb9df0f..7cc222c 100755 --- a/pipecat-agent/pipeline.py +++ b/pipecat-agent/pipeline.py @@ -1,32 +1,39 @@ """ -Pipecat pipeline -- v2 fully self-hosted. +Pipecat pipeline -- v2.1 optimized for low latency. -STT: Speaches (faster-whisper) via OpenAI-compatible /v1/audio/transcriptions -LLM: OpenClaw VoiceCoach agent via OpenAI-compatible gateway -TTS: Speaches (Kokoro 82M) via OpenAI-compatible /v1/audio/speech +STT: Speaches (faster-whisper) via /v1/audio/transcriptions (utterance-level) +LLM: OpenClaw VoiceCoach agent via OpenAI-compatible gateway (streaming) +TTS: Speaches (Kokoro 82M) via OpenAI-compatible /v1/audio/speech (sentence-level) + +Optimizations over v2.0: +- Sentence aggregator between LLM and TTS (stream first sentence while LLM generates rest) +- SpeechBrain runs async in background (non-blocking, one-turn-behind vocal state) +- VAD stop_secs reduced from 0.8 to 0.4 +- Whisper model default changed to large-v3-turbo (faster per-token on CPU/GPU) """ import asyncio import io import json import logging import os +import re import uuid import wave from datetime import datetime from pathlib import Path -from typing import AsyncGenerator, Optional +from typing import Optional import httpx from pipecat.frames.frames import ( - AudioRawFrame, EndFrame, ErrorFrame, Frame, TextFrame, TranscriptionFrame, - UserStartedSpeakingFrame, UserStoppedSpeakingFrame, + AudioRawFrame, EndFrame, TextFrame, TranscriptionFrame, + TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, + ErrorFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameProcessor, FrameDirection -from pipecat.services.ai_services import STTService from pipecat.services.openai import OpenAILLMService, OpenAITTSService from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams @@ -41,78 +48,130 @@ from prompt_builder import build_user_message, VocalState logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# SpeachesTTSService — Kokoro TTS via Speaches OpenAI-compatible API +# --------------------------------------------------------------------------- + +class SpeachesTTSService(OpenAITTSService): + """OpenAITTSService subclass that properly uses base_url and bypasses + the hardcoded VALID_VOICES dict for Kokoro voice IDs.""" + + def __init__(self, *, api_key=None, base_url=None, voice="alloy", + model="tts-1", sample_rate=24000, **kwargs): + from pipecat.services.ai_services import TTSService + TTSService.__init__(self, sample_rate=sample_rate, **kwargs) + self._settings = {"sample_rate": sample_rate} + self.set_model_name(model) + self.set_voice(voice) + from openai import AsyncOpenAI + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) + + async def run_tts(self, text: str): + logger.debug("TTS: [%s]", text[:80]) + try: + await self.start_ttfb_metrics() + async with self._client.audio.speech.with_streaming_response.create( + input=text or " ", + model=self.model_name, + voice=self._voice_id, + response_format="pcm", + ) as r: + if r.status_code != 200: + error = await r.text() + logger.error("TTS error: %s %s", r.status_code, error) + yield ErrorFrame(f"TTS error: {r.status_code}") + return + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + async for chunk in r.iter_bytes(8192): + if chunk: + await self.stop_ttfb_metrics() + yield TTSAudioRawFrame(chunk, 24000, 1) + yield TTSStoppedFrame() + except Exception as e: + logger.exception("TTS error: %s", e) + + +# --------------------------------------------------------------------------- +# Sentence Aggregator — buffers LLM streaming tokens into sentences for TTS +# --------------------------------------------------------------------------- + +SENTENCE_END = re.compile(r'[.!?]\s') + + +class SentenceAggregator(FrameProcessor): + """Buffers streaming TextFrames from LLM and emits one TextFrame per + complete sentence. This lets TTS start on the first sentence while + the LLM is still generating the rest.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._buffer = "" + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TextFrame): + self._buffer += frame.text + while True: + match = SENTENCE_END.search(self._buffer) + if match: + sentence = self._buffer[:match.end()].strip() + self._buffer = self._buffer[match.end():] + if sentence: + logger.info("Sentence -> TTS: %s", sentence[:80]) + await self.push_frame(TextFrame(text=sentence), direction) + else: + break + elif isinstance(frame, EndFrame): + if self._buffer.strip(): + logger.info("Flush -> TTS: %s", self._buffer.strip()[:80]) + await self.push_frame(TextFrame(text=self._buffer.strip()), direction) + self._buffer = "" + await self.push_frame(frame, direction) + else: + # Flush on any non-text frame (LLM response end, etc.) + if self._buffer.strip() and not isinstance(frame, AudioRawFrame): + logger.info("Flush -> TTS: %s", self._buffer.strip()[:80]) + await self.push_frame(TextFrame(text=self._buffer.strip()), direction) + self._buffer = "" + await self.push_frame(frame, direction) + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + SPEACHES_URL = os.getenv("SPEACHES_URL", "http://speaches:8000") -OPENCLAW_URL = os.getenv("OPENCLAW_URL", "http://voicecoach-openclaw:3100") +LLM_BASE_URL = os.getenv("LLM_BASE_URL", "http://host.docker.internal:11434/v1") +LLM_MODEL = os.getenv("LLM_MODEL", "llama3.1:8b") SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_URL", "http://speechbrain-service:8001") -WHISPER_MODEL = os.getenv("WHISPER_MODEL", "Systran/faster-whisper-large-v3-turbo") +WHISPER_MODEL = os.getenv("WHISPER_MODEL", "deepdml/faster-whisper-large-v3-turbo-ct2") +KOKORO_MODEL = os.getenv("KOKORO_MODEL", "speaches-ai/Kokoro-82M-v1.0-ONNX-fp16") KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_nova") -OPENCLAW_AGENT_ID = os.getenv("OPENCLAW_AGENT_ID", "voicecoach") CLIPS_DIR = Path(os.getenv("CLIPS_DIR", "/data/clips/pending")) UNCERTAINTY_THRESHOLD = 0.60 MIN_CLIP_DURATION = 1.0 -class SpeachesSTTService(STTService): - """STT via Speaches OpenAI-compatible /v1/audio/transcriptions endpoint. +# --------------------------------------------------------------------------- +# SpeachesSTTProcessor — buffers full utterance, sends once to Speaches +# --------------------------------------------------------------------------- - pipecat-ai 0.0.48 does not include OpenAISTTService; this thin wrapper - calls the Speaches HTTP API directly using the STTService base class. - """ +class SpeachesSTTProcessor(FrameProcessor): + """Buffers audio during speech, sends complete utterance to Speaches STT.""" def __init__(self, base_url: str, model: str, language: str = "en", **kwargs): super().__init__(**kwargs) self._base_url = base_url.rstrip("/") self._model = model self._language = language - self._http = httpx.AsyncClient(timeout=30.0) - - async def set_model(self, model: str): - self._model = model - - async def set_language(self, language): - self._language = str(language) - - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - if not audio: - return - # Wrap raw PCM in a WAV container for the API - buf = io.BytesIO() - with wave.open(buf, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(16000) - wf.writeframes(audio) - wav_bytes = buf.getvalue() - - try: - r = await self._http.post( - f"{self._base_url}/v1/audio/transcriptions", - files={"file": ("audio.wav", wav_bytes, "audio/wav")}, - data={"model": self._model, "language": self._language, "response_format": "json"}, - ) - r.raise_for_status() - text = r.json().get("text", "").strip() - if text: - logger.info("STT transcription: %s", text[:100]) - yield TranscriptionFrame(text=text, user_id="user", timestamp=datetime.utcnow().isoformat()) - except Exception as e: - logger.warning("Speaches STT error: %s", e) - yield ErrorFrame(error=str(e)) - - -class VocalStateInjector(FrameProcessor): - """Buffers audio per utterance, sends to SpeechBrain for analysis, - enriches TranscriptionFrame with [VOCAL_STATE] metadata, - and saves clips for active labeling.""" - - def __init__(self, session_id: str, **kwargs): - super().__init__(**kwargs) - self._session_id = session_id + self._http = httpx.AsyncClient(timeout=60.0) self._audio_buffer: list[bytes] = [] self._is_speaking = False - self._http = httpx.AsyncClient(timeout=3.0) async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -120,34 +179,113 @@ class VocalStateInjector(FrameProcessor): if isinstance(frame, UserStartedSpeakingFrame): self._is_speaking = True self._audio_buffer = [] - logger.info("VAD: user started speaking") + logger.info("STT: user started speaking") + await self.push_frame(frame, direction) + elif isinstance(frame, AudioRawFrame): + if self._is_speaking: + self._audio_buffer.append(frame.audio) + await self.push_frame(frame, direction) + elif isinstance(frame, UserStoppedSpeakingFrame): + self._is_speaking = False + logger.info("STT: user stopped, %d chunks", len(self._audio_buffer)) + if self._audio_buffer: + audio_bytes = b"".join(self._audio_buffer) + self._audio_buffer = [] + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + wf.writeframes(audio_bytes) + wav_bytes = buf.getvalue() + try: + logger.info("STT: sending %d bytes...", len(wav_bytes)) + r = await self._http.post( + f"{self._base_url}/v1/audio/transcriptions", + files={"file": ("utterance.wav", wav_bytes, "audio/wav")}, + data={ + "model": self._model, + "language": self._language, + "response_format": "json", + "vad_filter": "false", + }, + ) + r.raise_for_status() + text = r.json().get("text", "").strip() + if text: + logger.info("STT: %r", text) + await self.push_frame( + TranscriptionFrame(text=text, user_id="jeremy", + timestamp=datetime.utcnow().isoformat()), + direction, + ) + else: + logger.info("STT: empty transcription") + except Exception as e: + logger.warning("STT error: %s", e) + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + +# --------------------------------------------------------------------------- +# VocalStateInjector — async SpeechBrain (non-blocking, one-turn-behind) +# --------------------------------------------------------------------------- + +class VocalStateInjector(FrameProcessor): + """Enriches TranscriptionFrame with [VOCAL_STATE] from SpeechBrain. + Analysis runs in background — uses previous turn's result to avoid + blocking the critical path.""" + + def __init__(self, session_id: str, **kwargs): + super().__init__(**kwargs) + self._session_id = session_id + self._audio_buffer: list[bytes] = [] + self._is_speaking = False + self._http = httpx.AsyncClient(timeout=3.0) + self._last_vocal_state: Optional[VocalState] = None + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + self._is_speaking = True + self._audio_buffer = [] await self.push_frame(frame, direction) elif isinstance(frame, AudioRawFrame) and self._is_speaking: self._audio_buffer.append(frame.audio) await self.push_frame(frame, direction) elif isinstance(frame, UserStoppedSpeakingFrame): self._is_speaking = False - logger.info("VAD: user stopped speaking, buffer: %d chunks", len(self._audio_buffer)) await self.push_frame(frame, direction) elif isinstance(frame, TranscriptionFrame): - wav_bytes = self._flush_audio_to_wav() - self._audio_buffer = [] - - vocal_state = None - if wav_bytes: - vocal_state, analysis = await self._analyze(wav_bytes) - if analysis and self._clip_worth_saving(analysis, wav_bytes): - await self._save_clip(wav_bytes, analysis) - + # Use PREVIOUS turn's vocal state (non-blocking) + vocal_state = self._last_vocal_state enriched_text = build_user_message(frame.text, vocal_state) - logger.info("Enriched transcription: %s", enriched_text[:200]) + logger.info("-> LLM: %s", enriched_text[:120]) await self.push_frame( - TranscriptionFrame(text=enriched_text, user_id=frame.user_id, timestamp=frame.timestamp), + TranscriptionFrame(text=enriched_text, user_id=frame.user_id, + timestamp=frame.timestamp), direction, ) + # Fire background analysis for NEXT turn + wav_bytes = self._flush_audio_to_wav() + self._audio_buffer = [] + if wav_bytes: + asyncio.create_task(self._analyze_background(wav_bytes)) else: await self.push_frame(frame, direction) + async def _analyze_background(self, wav_bytes: bytes): + """Run SpeechBrain in background, store result for next turn.""" + try: + vocal_state, analysis = await self._analyze(wav_bytes) + self._last_vocal_state = vocal_state + if analysis and self._clip_worth_saving(analysis, wav_bytes): + await self._save_clip(wav_bytes, analysis) + except Exception as e: + logger.warning("Background SpeechBrain failed: %s", e) + def _flush_audio_to_wav(self) -> Optional[bytes]: if not self._audio_buffer: return None @@ -180,12 +318,11 @@ class VocalStateInjector(FrameProcessor): d, ) except Exception as e: - logger.warning("SpeechBrain unavailable: %s -- continuing without vocal state", e) + logger.warning("SpeechBrain unavailable: %s", e) return None, None def _clip_worth_saving(self, analysis: dict, wav_bytes: bytes) -> bool: - duration = len(wav_bytes) / (16000 * 2) - return duration >= MIN_CLIP_DURATION + return len(wav_bytes) / (16000 * 2) >= MIN_CLIP_DURATION async def _save_clip(self, wav_bytes: bytes, analysis: dict): try: @@ -193,8 +330,7 @@ class VocalStateInjector(FrameProcessor): clip_id = f"{self._session_id}_{uuid.uuid4().hex[:8]}" (CLIPS_DIR / f"{clip_id}.wav").write_bytes(wav_bytes) meta = { - "clip_id": clip_id, - "session_id": self._session_id, + "clip_id": clip_id, "session_id": self._session_id, "timestamp": datetime.utcnow().isoformat(), "duration_seconds": len(wav_bytes) / (16000 * 2), "speechbrain_emotion": analysis.get("emotion"), @@ -203,22 +339,70 @@ class VocalStateInjector(FrameProcessor): "pitch_mean_hz": analysis.get("pitch_mean_hz"), "energy_rms": analysis.get("energy_rms"), "speaking_rate_approx": analysis.get("speaking_rate_approx"), - "labeled": False, - "confirmed_label": None, + "labeled": False, "confirmed_label": None, "uncertain": analysis.get("emotion_confidence", 1.0) < UNCERTAINTY_THRESHOLD, } (CLIPS_DIR / f"{clip_id}.json").write_text(json.dumps(meta, indent=2)) - logger.info("Saved clip: %s", clip_id) except Exception as e: logger.warning("Failed to save clip: %s", e) +# --------------------------------------------------------------------------- +# Pre-warm +# --------------------------------------------------------------------------- + +async def _prewarm_speaches(): + """Pre-warm Whisper and Kokoro models so first request is fast.""" + client = httpx.AsyncClient(timeout=120.0) + try: + import numpy as np + sr = 16000 + audio = np.zeros(sr, dtype=np.int16) + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1); wf.setsampwidth(2); wf.setframerate(sr) + wf.writeframes(audio.tobytes()) + r = await client.post( + f"{SPEACHES_URL}/v1/audio/transcriptions", + files={"file": ("warmup.wav", buf.getvalue(), "audio/wav")}, + data={"model": WHISPER_MODEL, "language": "en", + "response_format": "json", "vad_filter": "false"}, + ) + logger.info("Pre-warm Whisper: %s", r.status_code) + r2 = await client.post( + f"{SPEACHES_URL}/v1/audio/speech", + json={"model": KOKORO_MODEL, "input": "Ready.", "voice": KOKORO_VOICE}, + ) + logger.info("Pre-warm Kokoro: %s", r2.status_code) + except Exception as e: + logger.warning("Pre-warm failed: %s", e) + finally: + await client.aclose() + + +# --------------------------------------------------------------------------- +# Pipeline assembly +# --------------------------------------------------------------------------- + async def run_voice_coach_pipeline( livekit_url: str, livekit_token: str, openclaw_token: str, ): session_id = datetime.utcnow().strftime("%Y%m%d-%H%M%S") + await _prewarm_speaches() + + # Load settings from token server + settings = {"confidence": 0.7, "min_volume": 0.3, "start_secs": 0.4, + "stop_secs": 0.5, "max_tokens": 150, "tts_voice": "af_nova"} + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.get("http://token-server:8080/settings") + if r.status_code == 200: + settings.update(r.json()) + logger.info("Loaded settings: %s", settings) + except Exception as e: + logger.warning("Could not load settings, using defaults: %s", e) transport = LiveKitTransport( url=livekit_url, @@ -229,41 +413,38 @@ async def run_voice_coach_pipeline( audio_out_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams( - confidence=0.5, - start_secs=0.2, - stop_secs=0.8, - min_volume=0.1, + confidence=float(settings["confidence"]), + start_secs=float(settings["start_secs"]), + stop_secs=float(settings["stop_secs"]), + min_volume=float(settings["min_volume"]), )), vad_audio_passthrough=True, ), ) - # STT: Speaches / faster-whisper via OpenAI-compatible API - stt = SpeachesSTTService( - base_url=SPEACHES_URL, - model=WHISPER_MODEL, - language="en", - ) - + stt = SpeachesSTTProcessor(base_url=SPEACHES_URL, model=WHISPER_MODEL, language="en") vocal_injector = VocalStateInjector(session_id=session_id) + sentence_agg = SentenceAggregator() + + from prompt_builder import build_system_prompt + system_prompt = build_system_prompt() - # LLM: OpenClaw VoiceCoach agent via OpenAI-compatible gateway llm = OpenAILLMService( - api_key=openclaw_token, - base_url=f"{OPENCLAW_URL}/v1", - model=f"openclaw/{OPENCLAW_AGENT_ID}", - max_tokens=512, + api_key="ollama", + base_url=LLM_BASE_URL, + model=LLM_MODEL, + max_tokens=int(settings["max_tokens"]), ) - - context = OpenAILLMContext(messages=[]) + context = OpenAILLMContext(messages=[ + {"role": "system", "content": system_prompt}, + ]) context_aggregator = llm.create_context_aggregator(context) - # TTS: Speaches / Kokoro via OpenAI-compatible API - tts = OpenAITTSService( + tts = SpeachesTTSService( api_key="speaches", base_url=f"{SPEACHES_URL}/v1", - model="kokoro", - voice=KOKORO_VOICE, + model=KOKORO_MODEL, + voice=settings.get("tts_voice", KOKORO_VOICE), ) pipeline = Pipeline([ @@ -272,25 +453,24 @@ async def run_voice_coach_pipeline( vocal_injector, context_aggregator.user(), llm, + sentence_agg, # Fix 1: aggregate sentences before TTS tts, transport.output(), context_aggregator.assistant(), ]) - task = PipelineTask( - pipeline, - params=PipelineParams(allow_interruptions=True), - ) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) runner = PipelineRunner() @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): logger.info("Participant joined: %s", participant) - await task.queue_frames([TextFrame("session_start")]) + await task.queue_frames([TTSSpeakFrame("VoiceCoach connected. How are you doing today?")]) @transport.event_handler("on_participant_left") async def on_participant_left(transport, participant, reason): - logger.info("Session ending: %s", session_id) - await task.queue_frames([EndFrame()]) + logger.info("Session ending: %s (reason: %s)", session_id, reason) + await task.cancel() await runner.run(task) + logger.info("Pipeline runner exited for session %s", session_id) diff --git a/pipecat-agent/piper_tts.py b/pipecat-agent/piper_tts.py new file mode 100755 index 0000000..83493de --- /dev/null +++ b/pipecat-agent/piper_tts.py @@ -0,0 +1,34 @@ +""" +Piper TTS client for Pipecat pipeline. +Calls the self-hosted Piper HTTP service. +""" +import io +import logging +import wave +import httpx + +logger = logging.getLogger(__name__) + + +class PiperTTSClient: + """HTTP client for the Piper TTS service.""" + + def __init__(self, base_url: str = "http://piper-tts:8003"): + self.base_url = base_url.rstrip("/") + self._client = httpx.AsyncClient(timeout=10.0) + + async def synthesize(self, text: str) -> bytes: + """Convert text to WAV audio bytes via Piper service.""" + try: + response = await self._client.post( + f"{self.base_url}/synthesize", + json={"text": text, "length_scale": 0.95, "sentence_silence": 0.15}, + ) + response.raise_for_status() + return response.content + except Exception as e: + logger.error(f"Piper TTS failed: {e}") + return b"" + + async def close(self): + await self._client.aclose() diff --git a/pipecat-agent/token_server.py b/pipecat-agent/token_server.py new file mode 100644 index 0000000..b055801 --- /dev/null +++ b/pipecat-agent/token_server.py @@ -0,0 +1,95 @@ +""" +Standalone LiveKit token server + settings API. +Runs independently from the pipeline so it never goes down when the pipeline crashes. +Settings are stored in a JSON file so the pipeline can read them on startup. +""" +import json +import os +from pathlib import Path +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel +from typing import Optional +from livekit import api as livekit_api + +LIVEKIT_API_KEY = os.environ["LIVEKIT_API_KEY"] +LIVEKIT_API_SECRET = os.environ["LIVEKIT_API_SECRET"] +LIVEKIT_PUBLIC_URL = os.getenv("LIVEKIT_PUBLIC_URL", "wss://voice.hobbs.farm") +SETTINGS_PATH = Path(os.getenv("SETTINGS_PATH", "/data/settings.json")) + +app = FastAPI(title="VoiceCoach Token + Settings Service") + +DEFAULTS = { + "confidence": 0.7, + "min_volume": 0.3, + "start_secs": 0.4, + "stop_secs": 0.5, + "max_tokens": 150, + "tts_voice": "af_nova", +} + + +def load_settings() -> dict: + try: + if SETTINGS_PATH.exists(): + return {**DEFAULTS, **json.loads(SETTINGS_PATH.read_text())} + except Exception: + pass + return {**DEFAULTS} + + +def save_settings(settings: dict): + SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + SETTINGS_PATH.write_text(json.dumps(settings, indent=2)) + + +class TokenRequest(BaseModel): + identity: str = "jeremy" + + +class TokenResponse(BaseModel): + token: str + url: str + + +class Settings(BaseModel): + confidence: Optional[float] = None + min_volume: Optional[float] = None + start_secs: Optional[float] = None + stop_secs: Optional[float] = None + max_tokens: Optional[int] = None + tts_voice: Optional[str] = None + + +@app.get("/health") +async def health(): + return {"status": "ok", "version": "2.1"} + + +@app.post("/token") +async def get_token(req: TokenRequest): + token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) + token.with_grants(livekit_api.VideoGrants( + room_join=True, room="voicecoach", + can_publish=True, can_subscribe=True, + )) + token.identity = req.identity + return TokenResponse(token=token.to_jwt(), url=LIVEKIT_PUBLIC_URL) + + +@app.get("/settings") +async def get_settings(): + return load_settings() + + +@app.post("/settings") +async def update_settings(s: Settings): + current = load_settings() + update = {k: v for k, v in s.dict().items() if v is not None} + current.update(update) + save_settings(current) + return current + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info") diff --git a/pipecat-agent/whisper_stt.py b/pipecat-agent/whisper_stt.py new file mode 100755 index 0000000..31f587e --- /dev/null +++ b/pipecat-agent/whisper_stt.py @@ -0,0 +1,35 @@ +""" +Local STT service using faster-whisper. +Replaces Deepgram with fully self-hosted speech-to-text. +""" +import io +import logging +import wave +import numpy as np +from faster_whisper import WhisperModel + +logger = logging.getLogger(__name__) + + +class LocalWhisperSTT: + """Wraps faster-whisper for use in the Pipecat pipeline.""" + + def __init__(self, model_size: str = "medium", device: str = "cpu", compute_type: str = "int8"): + logger.info(f"Loading faster-whisper model: {model_size} on {device}") + self.model = WhisperModel(model_size, device=device, compute_type=compute_type) + logger.info("faster-whisper model loaded.") + + def transcribe(self, audio_bytes: bytes, sample_rate: int = 16000) -> str: + """Transcribe raw PCM audio bytes to text.""" + audio_array = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0 + + segments, info = self.model.transcribe( + audio_array, + beam_size=5, + language="en", + vad_filter=True, + vad_parameters=dict(min_silence_duration_ms=500), + ) + + text = " ".join(seg.text.strip() for seg in segments) + return text diff --git a/web-client/app.js b/web-client/app.js index f4b36eb..b4f0d64 100755 --- a/web-client/app.js +++ b/web-client/app.js @@ -1,17 +1,119 @@ const LIVEKIT_URL = "wss://voice.hobbs.farm"; let room = null; +// --- Settings --- +const DEFAULTS = { + confidence: 0.7, min_volume: 0.3, start_secs: 0.4, + stop_secs: 0.5, max_tokens: 150, tts_voice: "af_nova" +}; + +function getSettings() { + try { + const saved = JSON.parse(localStorage.getItem("vc_settings")); + return saved ? { ...DEFAULTS, ...saved } : { ...DEFAULTS }; + } catch (e) { return { ...DEFAULTS }; } +} + +function loadSettingsUI() { + const s = getSettings(); + for (const [key, val] of Object.entries(s)) { + const input = document.getElementById("set-" + key); + const display = document.getElementById("val-" + key); + if (input) input.value = val; + if (display) display.textContent = val; + } + // Bind live value display + document.querySelectorAll(".settings-body input[type=range]").forEach(function(el) { + el.addEventListener("input", function() { + const key = el.id.replace("set-", ""); + document.getElementById("val-" + key).textContent = el.value; + }); + }); + var sel = document.getElementById("set-tts_voice"); + if (sel) sel.addEventListener("change", function() { + document.getElementById("val-tts_voice").textContent = sel.value; + }); +} + +function saveSettings() { + var s = {}; + for (var key of Object.keys(DEFAULTS)) { + var el = document.getElementById("set-" + key); + if (el) s[key] = el.tagName === "SELECT" ? el.value : parseFloat(el.value); + } + localStorage.setItem("vc_settings", JSON.stringify(s)); + // Push to server + fetch("/api/settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(s), + }).then(function(r) { + if (r.ok) log("Settings saved and applied"); + else log("Settings saved locally but server update failed", "warn"); + }).catch(function() { + log("Settings saved locally (server unreachable)", "warn"); + }); +} + +function resetSettings() { + localStorage.removeItem("vc_settings"); + loadSettingsUI(); + saveSettings(); + log("Settings reset to defaults"); +} + +function toggleSettings() { + var body = document.getElementById("settingsBody"); + var toggle = document.getElementById("settingsToggle"); + if (body.style.display === "none") { + body.style.display = "block"; + toggle.textContent = "-"; + } else { + body.style.display = "none"; + toggle.textContent = "+"; + } +} + +// --- Logging --- +function log(msg, level) { + var el = document.getElementById("logOutput"); + var time = new Date().toLocaleTimeString(); + var cls = level === "error" ? "log-error" : level === "warn" ? "log-warn" : "log-info"; + el.innerHTML += '