feat: initial commit — VoiceCoach real-time voice coaching pipeline

Services: Pipecat agent (LiveKit + Claude + Whisper STT + Piper TTS),
SpeechBrain analysis service, Piper TTS service, web client (nginx).
Includes LiveKit server config and docker-compose orchestration.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

pipecat-agent/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    libsndfile1 \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Pre-download faster-whisper medium model at build time
+RUN python -c "\
+from faster_whisper import WhisperModel; \
+WhisperModel('medium', device='cpu', compute_type='int8'); \
+print('Whisper model downloaded.')"
+
+COPY . .
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \
+    CMD python -c "print('pipeline process alive')" || exit 1
+
+CMD ["python", "main.py"]

pipecat-agent/main.py

@@ -0,0 +1,46 @@
+"""
+Entry point for the Pipecat VoiceCoach agent.
+Reads config from environment variables.
+"""
+import asyncio
+import logging
+import os
+
+import structlog
+from livekit import api as livekit_api
+
+from pipeline import run_voice_coach_pipeline
+
+logging.basicConfig(level=logging.INFO)
+logger = structlog.get_logger()
+
+
+async def main():
+    anthropic_key = os.environ["ANTHROPIC_API_KEY"]
+    livekit_api_key = os.environ["LIVEKIT_API_KEY"]
+    livekit_api_secret = os.environ["LIVEKIT_API_SECRET"]
+    livekit_url = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880")
+
+    # Generate LiveKit access token for this agent
+    token = livekit_api.AccessToken(livekit_api_key, livekit_api_secret)
+    token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach"))
+    token.identity = "voicecoach-agent"
+    livekit_token = token.to_jwt()
+
+    session_context = {
+        "baseline_context": "",
+        "session_history": "",
+        "current_mode": "coaching mode",
+    }
+
+    logger.info("Starting VoiceCoach pipeline", livekit_url=livekit_url)
+    await run_voice_coach_pipeline(
+        livekit_url=livekit_url,
+        livekit_token=livekit_token,
+        anthropic_key=anthropic_key,
+        session_context=session_context,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pipecat-agent/pipeline.py

@@ -0,0 +1,222 @@
+"""
+Main Pipecat pipeline assembly.
+Connects LiveKit -> Whisper STT -> SpeechBrain -> Claude -> Piper TTS -> LiveKit.
+Fully self-hosted STT/TTS (no external API dependencies except Anthropic).
+"""
+import asyncio
+import io
+import os
+import logging
+import wave
+import httpx
+from typing import Optional
+
+from pipecat.frames.frames import (
+    AudioRawFrame, TranscriptionFrame, TextFrame, EndFrame, TTSAudioRawFrame
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.anthropic import AnthropicLLMService
+from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams
+from pipecat.processors.frame_processor import FrameProcessor, FrameDirection
+from pipecat.vad.silero import SileroVADAnalyzer
+
+from prompt_builder import build_system_prompt, build_user_message, VocalState
+from whisper_stt import LocalWhisperSTT
+from piper_tts import PiperTTSClient
+
+logger = logging.getLogger(__name__)
+
+SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_SERVICE_URL", "http://speechbrain-service:8001")
+PIPER_URL = os.getenv("PIPER_SERVICE_URL", "http://piper-tts:8003")
+ANTHROPIC_MODEL = "claude-sonnet-4-6"
+WHISPER_MODEL = os.getenv("WHISPER_MODEL", "medium")
+
+
+class WhisperSTTProcessor(FrameProcessor):
+    """Buffers audio from VAD and transcribes with faster-whisper on end-of-speech."""
+
+    def __init__(self, whisper: LocalWhisperSTT, **kwargs):
+        super().__init__(**kwargs)
+        self._whisper = whisper
+        self._audio_buffer: list[bytes] = []
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, AudioRawFrame):
+            self._audio_buffer.append(frame.audio)
+        elif isinstance(frame, EndFrame):
+            # VAD detected end of speech -- transcribe
+            if self._audio_buffer:
+                audio_bytes = b"".join(self._audio_buffer)
+                self._audio_buffer = []
+                text = self._whisper.transcribe(audio_bytes)
+                if text.strip():
+                    await self.push_frame(
+                        TranscriptionFrame(text=text, user_id="jeremy", timestamp=""),
+                        direction,
+                    )
+            await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+
+class VocalStateInjector(FrameProcessor):
+    """Intercepts transcriptions, fetches vocal analysis from SpeechBrain,
+    and enriches the frame before passing to the LLM."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._current_audio: list[bytes] = []
+        self._http_client = httpx.AsyncClient(timeout=3.0)
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, AudioRawFrame):
+            self._current_audio.append(frame.audio)
+        elif isinstance(frame, TranscriptionFrame):
+            vocal_state = await self._analyze_buffered_audio()
+            self._current_audio = []
+            enriched_text = build_user_message(frame.text, vocal_state)
+            enriched_frame = TranscriptionFrame(
+                text=enriched_text, user_id=frame.user_id, timestamp=frame.timestamp,
+            )
+            await self.push_frame(enriched_frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _analyze_buffered_audio(self) -> Optional[VocalState]:
+        if not self._current_audio:
+            return None
+        try:
+            audio_bytes = b"".join(self._current_audio)
+            buf = io.BytesIO()
+            with wave.open(buf, "wb") as wf:
+                wf.setnchannels(1)
+                wf.setsampwidth(2)
+                wf.setframerate(16000)
+                wf.writeframes(audio_bytes)
+            wav_bytes = buf.getvalue()
+
+            response = await self._http_client.post(
+                f"{SPEECHBRAIN_URL}/analyze",
+                files={"audio": ("utterance.wav", wav_bytes, "audio/wav")},
+            )
+            response.raise_for_status()
+            data = response.json()
+            return VocalState(
+                emotion=data.get("emotion", "neutral"),
+                confidence=data.get("emotion_confidence", 0.0),
+                pitch_mean_hz=data.get("pitch_mean_hz"),
+                energy_rms=data.get("energy_rms", 0.0),
+                speaking_rate_approx=data.get("speaking_rate_approx", 0.0),
+                voiced_fraction=data.get("voiced_fraction", 0.0),
+            )
+        except Exception as e:
+            logger.warning(f"SpeechBrain analysis failed (continuing without): {e}")
+            return None
+
+
+class PiperTTSProcessor(FrameProcessor):
+    """Converts text frames to audio via Piper TTS service."""
+
+    def __init__(self, piper_client: PiperTTSClient, **kwargs):
+        super().__init__(**kwargs)
+        self._piper = piper_client
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, TextFrame):
+            wav_bytes = await self._piper.synthesize(frame.text)
+            if wav_bytes:
+                await self.push_frame(
+                    TTSAudioRawFrame(audio=wav_bytes, sample_rate=22050, num_channels=1),
+                    direction,
+                )
+        else:
+            await self.push_frame(frame, direction)
+
+
+async def run_voice_coach_pipeline(
+    livekit_url: str,
+    livekit_token: str,
+    anthropic_key: str,
+    session_context: dict,
+):
+    """Assemble and run the full voice coaching pipeline."""
+
+    whisper = LocalWhisperSTT(model_size=WHISPER_MODEL, device="cpu", compute_type="int8")
+    piper_client = PiperTTSClient(base_url=PIPER_URL)
+
+    transport = LiveKitTransport(
+        url=livekit_url,
+        token=livekit_token,
+        room_name="voicecoach",
+        params=LiveKitParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+        ),
+    )
+
+    whisper_stt = WhisperSTTProcessor(whisper)
+    vocal_injector = VocalStateInjector()
+
+    llm = AnthropicLLMService(
+        api_key=anthropic_key,
+        model=ANTHROPIC_MODEL,
+        max_tokens=512,
+    )
+
+    system_prompt = build_system_prompt(
+        baseline_context=session_context.get("baseline_context", ""),
+        session_history=session_context.get("session_history", ""),
+        current_mode=session_context.get("current_mode", "coaching mode"),
+    )
+
+    context = OpenAILLMContext(
+        messages=[{"role": "system", "content": system_prompt}]
+    )
+    context_aggregator = llm.create_context_aggregator(context)
+
+    piper_tts = PiperTTSProcessor(piper_client)
+
+    pipeline = Pipeline([
+        transport.input(),
+        whisper_stt,
+        vocal_injector,
+        context_aggregator.user(),
+        llm,
+        piper_tts,
+        transport.output(),
+        context_aggregator.assistant(),
+    ])
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(allow_interruptions=True),
+    )
+
+    runner = PipelineRunner()
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        logger.info(f"Participant joined: {participant['identity']}")
+        await task.queue_frames([
+            TextFrame("VoiceCoach connected. How are you doing today?")
+        ])
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        logger.info(f"Participant left: {participant['identity']}")
+        await task.queue_frames([EndFrame()])
+
+    try:
+        await runner.run(task)
+    finally:
+        await piper_client.close()

pipecat-agent/piper_tts.py

@@ -0,0 +1,34 @@
+"""
+Piper TTS client for Pipecat pipeline.
+Calls the self-hosted Piper HTTP service.
+"""
+import io
+import logging
+import wave
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class PiperTTSClient:
+    """HTTP client for the Piper TTS service."""
+
+    def __init__(self, base_url: str = "http://piper-tts:8003"):
+        self.base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(timeout=10.0)
+
+    async def synthesize(self, text: str) -> bytes:
+        """Convert text to WAV audio bytes via Piper service."""
+        try:
+            response = await self._client.post(
+                f"{self.base_url}/synthesize",
+                json={"text": text, "length_scale": 0.95, "sentence_silence": 0.15},
+            )
+            response.raise_for_status()
+            return response.content
+        except Exception as e:
+            logger.error(f"Piper TTS failed: {e}")
+            return b""
+
+    async def close(self):
+        await self._client.aclose()

pipecat-agent/prompt_builder.py

@@ -0,0 +1,76 @@
+"""
+Builds Claude prompts with embedded vocal state metadata.
+"""
+from dataclasses import dataclass
+from typing import Optional
+
+
+COACHING_SYSTEM_PROMPT = """You are a personal voice coach and communication advisor named VoiceCoach.
+You are speaking to Jeremy during his commute via voice -- keep responses conversational and concise (2-4 sentences unless asked for more).
+
+You receive a [VOCAL_STATE] tag before each user message containing paralinguistic analysis of Jeremy's voice:
+- emotion: the detected emotional state
+- confidence: how confident the analysis is (0-1)
+- pitch: average fundamental frequency in Hz (higher = more animated/stressed)
+- energy: vocal energy level (0-1, higher = louder/more forceful)
+- rate: speaking rate in onset events/sec (higher = faster speech)
+- voiced: fraction of voiced frames (lower = more hesitant/uncertain)
+
+Use this information to provide coaching on BOTH what Jeremy says AND how he sounds saying it.
+Important: Never quote the raw numbers. Translate them into natural observations.
+Example: "I notice some tension in your voice..." not "Your pitch was 215hz..."
+
+{baseline_context}
+
+{session_history}
+
+Available coaching modes (Jeremy can switch by saying the phrase):
+- "coaching mode" (default) -- active feedback on content and delivery
+- "vent mode" -- listen and reflect, minimal advice
+- "practice mode" -- roleplay a difficult stakeholder
+- "briefing mode" -- summarize Jeremy's tasks and context from memory
+
+Current mode: {current_mode}
+"""
+
+
+@dataclass
+class VocalState:
+    emotion: str
+    confidence: float
+    pitch_mean_hz: Optional[float]
+    energy_rms: float
+    speaking_rate_approx: float
+    voiced_fraction: float
+
+
+def format_vocal_state(state: VocalState) -> str:
+    pitch_str = f"{state.pitch_mean_hz:.0f}hz" if state.pitch_mean_hz else "unknown"
+    return (
+        f"[VOCAL_STATE "
+        f"emotion={state.emotion} "
+        f"confidence={state.confidence:.2f} "
+        f"pitch={pitch_str} "
+        f"energy={state.energy_rms:.2f} "
+        f"rate={state.speaking_rate_approx:.1f}ops "
+        f"voiced={state.voiced_fraction:.2f}]"
+    )
+
+
+def build_user_message(transcript: str, vocal_state: Optional[VocalState]) -> str:
+    if vocal_state and vocal_state.confidence > 0.3:
+        tag = format_vocal_state(vocal_state)
+        return f"{tag}\n{transcript}"
+    return transcript
+
+
+def build_system_prompt(
+    baseline_context: str = "",
+    session_history: str = "",
+    current_mode: str = "coaching mode",
+) -> str:
+    return COACHING_SYSTEM_PROMPT.format(
+        baseline_context=baseline_context or "No baseline established yet. First session: ask Jeremy to read a short neutral calibration script.",
+        session_history=session_history or "No previous session history.",
+        current_mode=current_mode,
+    )

pipecat-agent/requirements.txt

@@ -0,0 +1,10 @@
+pipecat-ai[livekit,silero,openai]==0.0.48
+anthropic==0.34.0
+faster-whisper==1.0.3
+requests==2.32.3
+httpx==0.27.0
+python-dotenv==1.0.1
+structlog==24.4.0
+numpy==1.26.4
+soundfile==0.12.1
+livekit-api==0.7.1

pipecat-agent/whisper_stt.py

@@ -0,0 +1,35 @@
+"""
+Local STT service using faster-whisper.
+Replaces Deepgram with fully self-hosted speech-to-text.
+"""
+import io
+import logging
+import wave
+import numpy as np
+from faster_whisper import WhisperModel
+
+logger = logging.getLogger(__name__)
+
+
+class LocalWhisperSTT:
+    """Wraps faster-whisper for use in the Pipecat pipeline."""
+
+    def __init__(self, model_size: str = "medium", device: str = "cpu", compute_type: str = "int8"):
+        logger.info(f"Loading faster-whisper model: {model_size} on {device}")
+        self.model = WhisperModel(model_size, device=device, compute_type=compute_type)
+        logger.info("faster-whisper model loaded.")
+
+    def transcribe(self, audio_bytes: bytes, sample_rate: int = 16000) -> str:
+        """Transcribe raw PCM audio bytes to text."""
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+
+        segments, info = self.model.transcribe(
+            audio_array,
+            beam_size=5,
+            language="en",
+            vad_filter=True,
+            vad_parameters=dict(min_silence_duration_ms=500),
+        )
+
+        text = " ".join(seg.text.strip() for seg in segments)
+        return text