feat: initial commit — VoiceCoach real-time voice coaching pipeline

Services: Pipecat agent (LiveKit + Claude + Whisper STT + Piper TTS),
SpeechBrain analysis service, Piper TTS service, web client (nginx).
Includes LiveKit server config and docker-compose orchestration.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -0,0 +1,9 @@
+# VoiceCoach Environment Variables
+# Copy to .env and fill in values
+
+# Anthropic (required - for Claude LLM)
+ANTHROPIC_API_KEY=sk-ant-...
+
+# LiveKit (auto-generated on first deploy if not set)
+LIVEKIT_API_KEY=voicecoach-key
+LIVEKIT_API_SECRET=your-livekit-secret-here

.gitignore

@@ -0,0 +1,51 @@
+# Environment secrets - never commit
+.env
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+.eggs/
+*.whl
+
+# Virtual environments
+venv/
+.venv/
+env/
+ENV/
+
+# Model weights and large binaries
+*.pt
+*.pth
+*.onnx
+*.bin
+*.safetensors
+models/
+
+# Piper voice models
+*.onnx.json
+
+# Logs
+*.log
+logs/
+
+# Docker
+.docker/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
+
+# Test artifacts
+.pytest_cache/
+.coverage
+htmlcov/

docker-compose.yml

@@ -0,0 +1,127 @@
+services:
+
+  # LiveKit WebRTC Server
+  livekit-server:
+    image: livekit/livekit-server:latest
+    container_name: voicecoach-livekit
+    restart: unless-stopped
+    command:
+      - --config
+      - /etc/livekit/livekit.yaml
+      - --keys
+      - "${LIVEKIT_API_KEY}: ${LIVEKIT_API_SECRET}"
+    ports:
+      - "7880:7880"
+      - "7881:7881"
+      - "50000-50100:50000-50100/udp"
+    volumes:
+      - ./livekit/livekit.yaml:/etc/livekit/livekit.yaml:ro
+    networks:
+      - voicecoach
+    mem_limit: 512M
+    labels:
+      - "com.centurylinklabs.watchtower.enable=false"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+        max-file: "3"
+
+  # SpeechBrain Emotion Analysis (internal only)
+  speechbrain-service:
+    build:
+      context: ./speechbrain-service
+    image: voicecoach-speechbrain:latest
+    container_name: voicecoach-speechbrain
+    restart: unless-stopped
+    volumes:
+      - /mnt/docker-data/configs/voicecoach/speechbrain-models:/app/models
+    environment:
+      - MODEL_DIR=/app/models
+    networks:
+      - voicecoach
+    mem_limit: 3G
+    memswap_limit: 3G
+    labels:
+      - "com.centurylinklabs.watchtower.enable=false"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+        max-file: "3"
+
+  # Piper TTS (internal only)
+  piper-tts:
+    build:
+      context: ./piper-service
+    image: voicecoach-piper:latest
+    container_name: voicecoach-piper
+    restart: unless-stopped
+    environment:
+      - PIPER_MODEL=/app/models/en_US-lessac-medium.onnx
+    networks:
+      - voicecoach
+    mem_limit: 512M
+    labels:
+      - "com.centurylinklabs.watchtower.enable=false"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+        max-file: "3"
+
+  # Pipecat Voice Pipeline Agent
+  pipecat-agent:
+    build:
+      context: ./pipecat-agent
+    image: voicecoach-pipecat:latest
+    container_name: voicecoach-pipecat
+    restart: unless-stopped
+    env_file:
+      - .env
+    environment:
+      - LIVEKIT_URL=ws://livekit-server:7880
+      - SPEECHBRAIN_SERVICE_URL=http://speechbrain-service:8001
+      - PIPER_SERVICE_URL=http://piper-tts:8003
+      - WHISPER_MODEL=medium
+    depends_on:
+      - livekit-server
+      - speechbrain-service
+      - piper-tts
+    networks:
+      - voicecoach
+    mem_limit: 3G
+    memswap_limit: 3G
+    labels:
+      - "com.centurylinklabs.watchtower.enable=false"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+        max-file: "3"
+
+  # Web Client (phone interface)
+  voicecoach-web:
+    build:
+      context: ./web-client
+    image: voicecoach-web:latest
+    container_name: voicecoach-web
+    restart: unless-stopped
+    ports:
+      - "8085:80"
+    depends_on:
+      - pipecat-agent
+    networks:
+      - voicecoach
+    mem_limit: 128M
+    labels:
+      - "com.centurylinklabs.watchtower.enable=false"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+networks:
+  voicecoach:
+    driver: bridge

livekit/livekit.yaml

@@ -0,0 +1,15 @@
+port: 7880
+rtc:
+  tcp_port: 7881
+  port_range_start: 50000
+  port_range_end: 60000
+  use_external_ip: true
+
+logging:
+  json: false
+  level: info
+
+room:
+  max_participants: 5
+  empty_timeout: 300
+  departure_timeout: 30

pipecat-agent/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    libsndfile1 \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Pre-download faster-whisper medium model at build time
+RUN python -c "\
+from faster_whisper import WhisperModel; \
+WhisperModel('medium', device='cpu', compute_type='int8'); \
+print('Whisper model downloaded.')"
+
+COPY . .
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \
+    CMD python -c "print('pipeline process alive')" || exit 1
+
+CMD ["python", "main.py"]

pipecat-agent/main.py

@@ -0,0 +1,46 @@
+"""
+Entry point for the Pipecat VoiceCoach agent.
+Reads config from environment variables.
+"""
+import asyncio
+import logging
+import os
+
+import structlog
+from livekit import api as livekit_api
+
+from pipeline import run_voice_coach_pipeline
+
+logging.basicConfig(level=logging.INFO)
+logger = structlog.get_logger()
+
+
+async def main():
+    anthropic_key = os.environ["ANTHROPIC_API_KEY"]
+    livekit_api_key = os.environ["LIVEKIT_API_KEY"]
+    livekit_api_secret = os.environ["LIVEKIT_API_SECRET"]
+    livekit_url = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880")
+
+    # Generate LiveKit access token for this agent
+    token = livekit_api.AccessToken(livekit_api_key, livekit_api_secret)
+    token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach"))
+    token.identity = "voicecoach-agent"
+    livekit_token = token.to_jwt()
+
+    session_context = {
+        "baseline_context": "",
+        "session_history": "",
+        "current_mode": "coaching mode",
+    }
+
+    logger.info("Starting VoiceCoach pipeline", livekit_url=livekit_url)
+    await run_voice_coach_pipeline(
+        livekit_url=livekit_url,
+        livekit_token=livekit_token,
+        anthropic_key=anthropic_key,
+        session_context=session_context,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pipecat-agent/pipeline.py

@@ -0,0 +1,222 @@
+"""
+Main Pipecat pipeline assembly.
+Connects LiveKit -> Whisper STT -> SpeechBrain -> Claude -> Piper TTS -> LiveKit.
+Fully self-hosted STT/TTS (no external API dependencies except Anthropic).
+"""
+import asyncio
+import io
+import os
+import logging
+import wave
+import httpx
+from typing import Optional
+
+from pipecat.frames.frames import (
+    AudioRawFrame, TranscriptionFrame, TextFrame, EndFrame, TTSAudioRawFrame
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.anthropic import AnthropicLLMService
+from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams
+from pipecat.processors.frame_processor import FrameProcessor, FrameDirection
+from pipecat.vad.silero import SileroVADAnalyzer
+
+from prompt_builder import build_system_prompt, build_user_message, VocalState
+from whisper_stt import LocalWhisperSTT
+from piper_tts import PiperTTSClient
+
+logger = logging.getLogger(__name__)
+
+SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_SERVICE_URL", "http://speechbrain-service:8001")
+PIPER_URL = os.getenv("PIPER_SERVICE_URL", "http://piper-tts:8003")
+ANTHROPIC_MODEL = "claude-sonnet-4-6"
+WHISPER_MODEL = os.getenv("WHISPER_MODEL", "medium")
+
+
+class WhisperSTTProcessor(FrameProcessor):
+    """Buffers audio from VAD and transcribes with faster-whisper on end-of-speech."""
+
+    def __init__(self, whisper: LocalWhisperSTT, **kwargs):
+        super().__init__(**kwargs)
+        self._whisper = whisper
+        self._audio_buffer: list[bytes] = []
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, AudioRawFrame):
+            self._audio_buffer.append(frame.audio)
+        elif isinstance(frame, EndFrame):
+            # VAD detected end of speech -- transcribe
+            if self._audio_buffer:
+                audio_bytes = b"".join(self._audio_buffer)
+                self._audio_buffer = []
+                text = self._whisper.transcribe(audio_bytes)
+                if text.strip():
+                    await self.push_frame(
+                        TranscriptionFrame(text=text, user_id="jeremy", timestamp=""),
+                        direction,
+                    )
+            await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+
+class VocalStateInjector(FrameProcessor):
+    """Intercepts transcriptions, fetches vocal analysis from SpeechBrain,
+    and enriches the frame before passing to the LLM."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._current_audio: list[bytes] = []
+        self._http_client = httpx.AsyncClient(timeout=3.0)
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, AudioRawFrame):
+            self._current_audio.append(frame.audio)
+        elif isinstance(frame, TranscriptionFrame):
+            vocal_state = await self._analyze_buffered_audio()
+            self._current_audio = []
+            enriched_text = build_user_message(frame.text, vocal_state)
+            enriched_frame = TranscriptionFrame(
+                text=enriched_text, user_id=frame.user_id, timestamp=frame.timestamp,
+            )
+            await self.push_frame(enriched_frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _analyze_buffered_audio(self) -> Optional[VocalState]:
+        if not self._current_audio:
+            return None
+        try:
+            audio_bytes = b"".join(self._current_audio)
+            buf = io.BytesIO()
+            with wave.open(buf, "wb") as wf:
+                wf.setnchannels(1)
+                wf.setsampwidth(2)
+                wf.setframerate(16000)
+                wf.writeframes(audio_bytes)
+            wav_bytes = buf.getvalue()
+
+            response = await self._http_client.post(
+                f"{SPEECHBRAIN_URL}/analyze",
+                files={"audio": ("utterance.wav", wav_bytes, "audio/wav")},
+            )
+            response.raise_for_status()
+            data = response.json()
+            return VocalState(
+                emotion=data.get("emotion", "neutral"),
+                confidence=data.get("emotion_confidence", 0.0),
+                pitch_mean_hz=data.get("pitch_mean_hz"),
+                energy_rms=data.get("energy_rms", 0.0),
+                speaking_rate_approx=data.get("speaking_rate_approx", 0.0),
+                voiced_fraction=data.get("voiced_fraction", 0.0),
+            )
+        except Exception as e:
+            logger.warning(f"SpeechBrain analysis failed (continuing without): {e}")
+            return None
+
+
+class PiperTTSProcessor(FrameProcessor):
+    """Converts text frames to audio via Piper TTS service."""
+
+    def __init__(self, piper_client: PiperTTSClient, **kwargs):
+        super().__init__(**kwargs)
+        self._piper = piper_client
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, TextFrame):
+            wav_bytes = await self._piper.synthesize(frame.text)
+            if wav_bytes:
+                await self.push_frame(
+                    TTSAudioRawFrame(audio=wav_bytes, sample_rate=22050, num_channels=1),
+                    direction,
+                )
+        else:
+            await self.push_frame(frame, direction)
+
+
+async def run_voice_coach_pipeline(
+    livekit_url: str,
+    livekit_token: str,
+    anthropic_key: str,
+    session_context: dict,
+):
+    """Assemble and run the full voice coaching pipeline."""
+
+    whisper = LocalWhisperSTT(model_size=WHISPER_MODEL, device="cpu", compute_type="int8")
+    piper_client = PiperTTSClient(base_url=PIPER_URL)
+
+    transport = LiveKitTransport(
+        url=livekit_url,
+        token=livekit_token,
+        room_name="voicecoach",
+        params=LiveKitParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+        ),
+    )
+
+    whisper_stt = WhisperSTTProcessor(whisper)
+    vocal_injector = VocalStateInjector()
+
+    llm = AnthropicLLMService(
+        api_key=anthropic_key,
+        model=ANTHROPIC_MODEL,
+        max_tokens=512,
+    )
+
+    system_prompt = build_system_prompt(
+        baseline_context=session_context.get("baseline_context", ""),
+        session_history=session_context.get("session_history", ""),
+        current_mode=session_context.get("current_mode", "coaching mode"),
+    )
+
+    context = OpenAILLMContext(
+        messages=[{"role": "system", "content": system_prompt}]
+    )
+    context_aggregator = llm.create_context_aggregator(context)
+
+    piper_tts = PiperTTSProcessor(piper_client)
+
+    pipeline = Pipeline([
+        transport.input(),
+        whisper_stt,
+        vocal_injector,
+        context_aggregator.user(),
+        llm,
+        piper_tts,
+        transport.output(),
+        context_aggregator.assistant(),
+    ])
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(allow_interruptions=True),
+    )
+
+    runner = PipelineRunner()
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        logger.info(f"Participant joined: {participant['identity']}")
+        await task.queue_frames([
+            TextFrame("VoiceCoach connected. How are you doing today?")
+        ])
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        logger.info(f"Participant left: {participant['identity']}")
+        await task.queue_frames([EndFrame()])
+
+    try:
+        await runner.run(task)
+    finally:
+        await piper_client.close()

pipecat-agent/piper_tts.py

@@ -0,0 +1,34 @@
+"""
+Piper TTS client for Pipecat pipeline.
+Calls the self-hosted Piper HTTP service.
+"""
+import io
+import logging
+import wave
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class PiperTTSClient:
+    """HTTP client for the Piper TTS service."""
+
+    def __init__(self, base_url: str = "http://piper-tts:8003"):
+        self.base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(timeout=10.0)
+
+    async def synthesize(self, text: str) -> bytes:
+        """Convert text to WAV audio bytes via Piper service."""
+        try:
+            response = await self._client.post(
+                f"{self.base_url}/synthesize",
+                json={"text": text, "length_scale": 0.95, "sentence_silence": 0.15},
+            )
+            response.raise_for_status()
+            return response.content
+        except Exception as e:
+            logger.error(f"Piper TTS failed: {e}")
+            return b""
+
+    async def close(self):
+        await self._client.aclose()

pipecat-agent/prompt_builder.py

@@ -0,0 +1,76 @@
+"""
+Builds Claude prompts with embedded vocal state metadata.
+"""
+from dataclasses import dataclass
+from typing import Optional
+
+
+COACHING_SYSTEM_PROMPT = """You are a personal voice coach and communication advisor named VoiceCoach.
+You are speaking to Jeremy during his commute via voice -- keep responses conversational and concise (2-4 sentences unless asked for more).
+
+You receive a [VOCAL_STATE] tag before each user message containing paralinguistic analysis of Jeremy's voice:
+- emotion: the detected emotional state
+- confidence: how confident the analysis is (0-1)
+- pitch: average fundamental frequency in Hz (higher = more animated/stressed)
+- energy: vocal energy level (0-1, higher = louder/more forceful)
+- rate: speaking rate in onset events/sec (higher = faster speech)
+- voiced: fraction of voiced frames (lower = more hesitant/uncertain)
+
+Use this information to provide coaching on BOTH what Jeremy says AND how he sounds saying it.
+Important: Never quote the raw numbers. Translate them into natural observations.
+Example: "I notice some tension in your voice..." not "Your pitch was 215hz..."
+
+{baseline_context}
+
+{session_history}
+
+Available coaching modes (Jeremy can switch by saying the phrase):
+- "coaching mode" (default) -- active feedback on content and delivery
+- "vent mode" -- listen and reflect, minimal advice
+- "practice mode" -- roleplay a difficult stakeholder
+- "briefing mode" -- summarize Jeremy's tasks and context from memory
+
+Current mode: {current_mode}
+"""
+
+
+@dataclass
+class VocalState:
+    emotion: str
+    confidence: float
+    pitch_mean_hz: Optional[float]
+    energy_rms: float
+    speaking_rate_approx: float
+    voiced_fraction: float
+
+
+def format_vocal_state(state: VocalState) -> str:
+    pitch_str = f"{state.pitch_mean_hz:.0f}hz" if state.pitch_mean_hz else "unknown"
+    return (
+        f"[VOCAL_STATE "
+        f"emotion={state.emotion} "
+        f"confidence={state.confidence:.2f} "
+        f"pitch={pitch_str} "
+        f"energy={state.energy_rms:.2f} "
+        f"rate={state.speaking_rate_approx:.1f}ops "
+        f"voiced={state.voiced_fraction:.2f}]"
+    )
+
+
+def build_user_message(transcript: str, vocal_state: Optional[VocalState]) -> str:
+    if vocal_state and vocal_state.confidence > 0.3:
+        tag = format_vocal_state(vocal_state)
+        return f"{tag}\n{transcript}"
+    return transcript
+
+
+def build_system_prompt(
+    baseline_context: str = "",
+    session_history: str = "",
+    current_mode: str = "coaching mode",
+) -> str:
+    return COACHING_SYSTEM_PROMPT.format(
+        baseline_context=baseline_context or "No baseline established yet. First session: ask Jeremy to read a short neutral calibration script.",
+        session_history=session_history or "No previous session history.",
+        current_mode=current_mode,
+    )

pipecat-agent/requirements.txt

@@ -0,0 +1,10 @@
+pipecat-ai[livekit,silero,openai]==0.0.48
+anthropic==0.34.0
+faster-whisper==1.0.3
+requests==2.32.3
+httpx==0.27.0
+python-dotenv==1.0.1
+structlog==24.4.0
+numpy==1.26.4
+soundfile==0.12.1
+livekit-api==0.7.1

pipecat-agent/whisper_stt.py

@@ -0,0 +1,35 @@
+"""
+Local STT service using faster-whisper.
+Replaces Deepgram with fully self-hosted speech-to-text.
+"""
+import io
+import logging
+import wave
+import numpy as np
+from faster_whisper import WhisperModel
+
+logger = logging.getLogger(__name__)
+
+
+class LocalWhisperSTT:
+    """Wraps faster-whisper for use in the Pipecat pipeline."""
+
+    def __init__(self, model_size: str = "medium", device: str = "cpu", compute_type: str = "int8"):
+        logger.info(f"Loading faster-whisper model: {model_size} on {device}")
+        self.model = WhisperModel(model_size, device=device, compute_type=compute_type)
+        logger.info("faster-whisper model loaded.")
+
+    def transcribe(self, audio_bytes: bytes, sample_rate: int = 16000) -> str:
+        """Transcribe raw PCM audio bytes to text."""
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+
+        segments, info = self.model.transcribe(
+            audio_array,
+            beam_size=5,
+            language="en",
+            vad_filter=True,
+            vad_parameters=dict(min_silence_duration_ms=500),
+        )
+
+        text = " ".join(seg.text.strip() for seg in segments)
+        return text

piper-service/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    libsndfile1 \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Download Piper voice model at build time from HuggingFace (lessac medium - good quality, fast on CPU)
+RUN mkdir -p /app/models && \
+    wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx?download=true" \
+        -O /app/models/en_US-lessac-medium.onnx && \
+    wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json?download=true" \
+        -O /app/models/en_US-lessac-medium.onnx.json
+
+COPY . .
+
+EXPOSE 8003
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \
+    CMD python -c "import httpx; httpx.get('http://localhost:8003/health').raise_for_status()"
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8003", "--workers", "1"]

piper-service/main.py

@@ -0,0 +1,72 @@
+"""
+Piper TTS HTTP service.
+Self-hosted text-to-speech, internal network only.
+"""
+import io
+import logging
+import os
+import wave
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import Response
+from pydantic import BaseModel
+import piper
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+MODEL_PATH = os.getenv("PIPER_MODEL", "/app/models/en_US-lessac-medium.onnx")
+voice = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global voice
+    logger.info(f"Loading Piper model: {MODEL_PATH}")
+    voice = piper.PiperVoice.load(MODEL_PATH)
+    logger.info("Piper TTS ready.")
+    yield
+    logger.info("Shutting down.")
+
+
+app = FastAPI(title="Piper TTS Service", lifespan=lifespan)
+
+
+class TTSRequest(BaseModel):
+    text: str
+    speaker_id: int = 0
+    length_scale: float = 1.0
+    sentence_silence: float = 0.2
+
+
+@app.get("/health")
+async def health():
+    if voice is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    return {"status": "ok", "model": os.path.basename(MODEL_PATH)}
+
+
+@app.post("/synthesize")
+async def synthesize(req: TTSRequest):
+    if voice is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+
+    if not req.text.strip():
+        raise HTTPException(status_code=400, detail="Empty text")
+
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        voice.synthesize(
+            req.text,
+            wf,
+            speaker_id=req.speaker_id,
+            length_scale=req.length_scale,
+            sentence_silence=req.sentence_silence,
+        )
+
+    return Response(
+        content=buf.getvalue(),
+        media_type="audio/wav",
+        headers={"Content-Disposition": "inline; filename=speech.wav"},
+    )

piper-service/requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.0
+piper-tts==1.2.0
+httpx==0.27.0
+numpy==1.26.4

speechbrain-service/Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    libsndfile1 \
+    ffmpeg \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Pre-download the SpeechBrain model at build time
+RUN python -c "\
+from speechbrain.inference.interfaces import foreign_class; \
+foreign_class( \
+    source='speechbrain/emotion-recognition-wav2vec2-IEMOCAP', \
+    pymodule_file='custom_interface.py', \
+    classname='CustomEncoderWav2vec2Classifier', \
+    savedir='/app/models', \
+    run_opts={'device': 'cpu'}, \
+); \
+print('Model downloaded successfully.')"
+
+COPY . .
+
+EXPOSE 8001
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=90s \
+    CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()"
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"]

speechbrain-service/analyzer.py

@@ -0,0 +1,119 @@
+"""
+SpeechBrain + librosa vocal analysis.
+Performs emotion classification and acoustic feature extraction from audio.
+"""
+import io
+import logging
+import numpy as np
+import soundfile as sf
+import librosa
+from speechbrain.inference.interfaces import foreign_class
+from dataclasses import dataclass, asdict
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+SAMPLE_RATE = 16000
+MIN_DURATION_SECONDS = 0.5
+
+
+@dataclass
+class VocalAnalysis:
+    emotion: str
+    emotion_confidence: float
+    emotion_scores: dict
+    pitch_mean_hz: Optional[float]
+    pitch_variance: Optional[float]
+    energy_rms: float
+    speaking_rate_approx: float
+    voiced_fraction: float
+    duration_seconds: float
+    analysis_error: Optional[str] = None
+
+
+class VocalAnalyzer:
+    """Loads models once at startup and provides analysis per audio chunk."""
+
+    def __init__(self, model_dir: str = "/app/models"):
+        logger.info("Loading SpeechBrain emotion model...")
+        self.classifier = foreign_class(
+            source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP",
+            pymodule_file="custom_interface.py",
+            classname="CustomEncoderWav2vec2Classifier",
+            savedir=model_dir,
+            run_opts={"device": "cpu"},
+        )
+        logger.info("SpeechBrain model loaded successfully.")
+
+    def analyze(self, audio_bytes: bytes, content_type: str = "audio/wav") -> VocalAnalysis:
+        try:
+            audio_buf = io.BytesIO(audio_bytes)
+            audio, sr = sf.read(audio_buf, dtype="float32")
+
+            if audio.ndim > 1:
+                audio = audio.mean(axis=1)
+
+            if sr != SAMPLE_RATE:
+                audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+                sr = SAMPLE_RATE
+
+            duration = len(audio) / sr
+
+            if duration < MIN_DURATION_SECONDS:
+                return VocalAnalysis(
+                    emotion="neutral", emotion_confidence=0.0, emotion_scores={},
+                    pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0,
+                    speaking_rate_approx=0.0, voiced_fraction=0.0,
+                    duration_seconds=duration,
+                    analysis_error=f"Audio too short: {duration:.2f}s < {MIN_DURATION_SECONDS}s minimum",
+                )
+
+            import tempfile, os
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+                sf.write(tmp.name, audio, sr)
+                tmp_path = tmp.name
+
+            try:
+                out_prob, score, index, text_lab = self.classifier.classify_file(tmp_path)
+                emotion = text_lab[0] if isinstance(text_lab, list) else text_lab
+                confidence = float(score[0]) if hasattr(score, "__iter__") else float(score)
+                labels = ["neu", "hap", "sad", "ang", "fru"]
+                prob_list = out_prob[0].tolist() if hasattr(out_prob[0], "tolist") else list(out_prob[0])
+                emotion_scores = {l: round(p, 4) for l, p in zip(labels, prob_list)}
+            finally:
+                os.unlink(tmp_path)
+
+            f0, voiced_flag, _ = librosa.pyin(
+                audio, fmin=librosa.note_to_hz("C2"), fmax=librosa.note_to_hz("C7"), sr=sr,
+            )
+
+            voiced_f0 = f0[voiced_flag] if voiced_flag is not None else np.array([])
+            pitch_mean = float(np.mean(voiced_f0)) if len(voiced_f0) > 0 else None
+            pitch_var = float(np.var(voiced_f0)) if len(voiced_f0) > 1 else None
+            voiced_fraction = float(np.sum(voiced_flag) / len(voiced_flag)) if voiced_flag is not None else 0.0
+
+            rms = librosa.feature.rms(y=audio)
+            energy_rms = float(np.mean(rms))
+
+            onset_frames = librosa.onset.onset_detect(y=audio, sr=sr)
+            speaking_rate = len(onset_frames) / duration if duration > 0 else 0.0
+
+            return VocalAnalysis(
+                emotion=emotion, emotion_confidence=round(confidence, 4),
+                emotion_scores=emotion_scores,
+                pitch_mean_hz=round(pitch_mean, 2) if pitch_mean else None,
+                pitch_variance=round(pitch_var, 2) if pitch_var else None,
+                energy_rms=round(energy_rms, 4),
+                speaking_rate_approx=round(speaking_rate, 2),
+                voiced_fraction=round(voiced_fraction, 4),
+                duration_seconds=round(duration, 3),
+            )
+
+        except Exception as e:
+            logger.exception(f"Analysis failed: {e}")
+            return VocalAnalysis(
+                emotion="neutral", emotion_confidence=0.0, emotion_scores={},
+                pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0,
+                speaking_rate_approx=0.0, voiced_fraction=0.0,
+                duration_seconds=0.0, analysis_error=str(e),
+            )

speechbrain-service/main.py

@@ -0,0 +1,66 @@
+"""
+FastAPI service wrapping VocalAnalyzer.
+Internal-only -- not exposed via Traefik.
+"""
+import logging
+import os
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, UploadFile, File, HTTPException
+from fastapi.responses import JSONResponse
+from dataclasses import asdict
+
+from analyzer import VocalAnalyzer
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+MODEL_DIR = os.getenv("MODEL_DIR", "/app/models")
+analyzer: VocalAnalyzer = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global analyzer
+    logger.info("Starting VocalAnalyzer...")
+    analyzer = VocalAnalyzer(model_dir=MODEL_DIR)
+    logger.info("VocalAnalyzer ready.")
+    yield
+    logger.info("Shutting down.")
+
+
+app = FastAPI(title="SpeechBrain Vocal Analysis Service", lifespan=lifespan)
+
+
+@app.get("/health")
+async def health():
+    if analyzer is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    return {"status": "ok", "model": "emotion-recognition-wav2vec2-IEMOCAP"}
+
+
+@app.post("/analyze")
+async def analyze(audio: UploadFile = File(...)):
+    if analyzer is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+
+    content = await audio.read()
+    if len(content) == 0:
+        raise HTTPException(status_code=400, detail="Empty audio file")
+
+    result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav")
+    return JSONResponse(content=asdict(result))
+
+
+@app.post("/calibrate")
+async def calibrate(audio: UploadFile = File(...)):
+    if analyzer is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+
+    content = await audio.read()
+    if len(content) == 0:
+        raise HTTPException(status_code=400, detail="Empty audio file")
+
+    result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav")
+    result_dict = asdict(result)
+    result_dict["is_baseline"] = True
+    return JSONResponse(content=result_dict)

speechbrain-service/requirements.txt

@@ -0,0 +1,12 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.0
+speechbrain==1.0.0
+huggingface_hub==0.19.4
+transformers==4.40.2
+torch==2.3.0
+torchaudio==2.3.0
+librosa==0.10.2
+soundfile==0.12.1
+numpy==1.26.4
+httpx==0.27.0
+python-multipart==0.0.9

web-client/Dockerfile

@@ -0,0 +1,8 @@
+FROM nginx:alpine
+
+COPY index.html /usr/share/nginx/html/
+COPY app.js /usr/share/nginx/html/
+COPY style.css /usr/share/nginx/html/
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+
+EXPOSE 80

web-client/app.js

@@ -0,0 +1,77 @@
+const LIVEKIT_URL = "wss://voice.hobbs.farm";
+let room = null;
+
+async function connect() {
+  setStatus("connecting", "Connecting...");
+  try {
+    // Request a token from the token service
+    const response = await fetch("/api/token", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ identity: "jeremy-" + Date.now() }),
+    });
+    if (!response.ok) throw new Error("Failed to get token");
+    const { token } = await response.json();
+
+    room = new LivekitClient.Room({
+      audioCaptureDefaults: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      },
+    });
+
+    room.on(LivekitClient.RoomEvent.Connected, () => {
+      setStatus("connected", "Connected");
+      document.getElementById("connectBtn").style.display = "none";
+      document.getElementById("disconnectBtn").style.display = "block";
+    });
+
+    room.on(LivekitClient.RoomEvent.Disconnected, () => {
+      setStatus("disconnected", "Disconnected");
+      document.getElementById("connectBtn").style.display = "block";
+      document.getElementById("disconnectBtn").style.display = "none";
+    });
+
+    room.on(LivekitClient.RoomEvent.TrackSubscribed, (track) => {
+      if (track.kind === LivekitClient.Track.Kind.Audio) {
+        const element = track.attach();
+        document.body.appendChild(element);
+        element.play();
+      }
+    });
+
+    await room.connect(LIVEKIT_URL, token);
+    await room.localParticipant.setMicrophoneEnabled(true);
+  } catch (err) {
+    setStatus("disconnected", "Error: " + err.message);
+    console.error(err);
+  }
+}
+
+async function disconnect() {
+  if (room) {
+    await room.disconnect();
+    room = null;
+  }
+}
+
+function setMode(mode) {
+  document.getElementById("modeDisplay").textContent = "Mode: " + mode;
+  if (room && room.localParticipant) {
+    const encoder = new TextEncoder();
+    room.localParticipant.publishData(
+      encoder.encode(JSON.stringify({ type: "mode_change", mode })),
+      { reliable: true }
+    );
+  }
+}
+
+function setStatus(state, text) {
+  const el = document.getElementById("status");
+  el.className = "status " + state;
+  el.textContent = text;
+}
+
+document.getElementById("connectBtn").addEventListener("click", connect);
+document.getElementById("disconnectBtn").addEventListener("click", disconnect);

web-client/index.html

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
+  <title>VoiceCoach</title>
+  <link rel="stylesheet" href="style.css">
+</head>
+<body>
+  <div class="container">
+    <div class="header">
+      <h1>VoiceCoach</h1>
+      <div id="status" class="status disconnected">Disconnected</div>
+    </div>
+
+    <div class="controls">
+      <button id="connectBtn" class="btn btn-primary">Start Session</button>
+      <button id="disconnectBtn" class="btn btn-danger" style="display:none">End Session</button>
+    </div>
+
+    <div id="modeDisplay" class="mode-display">Mode: coaching</div>
+
+    <div id="transcript" class="transcript">
+      <p class="hint">Press "Start Session" to connect your voice.</p>
+    </div>
+
+    <div class="mode-buttons">
+      <button class="mode-btn" onclick="setMode('coaching mode')">Coaching</button>
+      <button class="mode-btn" onclick="setMode('vent mode')">Vent</button>
+      <button class="mode-btn" onclick="setMode('practice mode')">Practice</button>
+      <button class="mode-btn" onclick="setMode('briefing mode')">Briefing</button>
+    </div>
+  </div>
+
+  <script src="https://cdn.jsdelivr.net/npm/livekit-client/dist/livekit-client.umd.min.js"></script>
+  <script src="app.js"></script>
+</body>
+</html>

web-client/nginx.conf

@@ -0,0 +1,15 @@
+server {
+    listen 80;
+    root /usr/share/nginx/html;
+    index index.html;
+
+    location /api/ {
+        proxy_pass http://pipecat-agent:8080/;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+    }
+
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+}

web-client/style.css

@@ -0,0 +1,88 @@
+* { margin: 0; padding: 0; box-sizing: border-box; }
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+  background: #1a1a2e;
+  color: #eee;
+  min-height: 100vh;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+.container {
+  max-width: 420px;
+  width: 100%;
+  padding: 24px;
+}
+
+.header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 24px;
+}
+
+h1 { font-size: 24px; font-weight: 600; }
+
+.status {
+  padding: 4px 12px;
+  border-radius: 12px;
+  font-size: 12px;
+  font-weight: 600;
+  text-transform: uppercase;
+}
+.status.disconnected { background: #444; color: #aaa; }
+.status.connecting { background: #553; color: #ffa; }
+.status.connected { background: #253; color: #5f5; }
+
+.controls { margin-bottom: 20px; }
+
+.btn {
+  width: 100%;
+  padding: 14px;
+  border: none;
+  border-radius: 8px;
+  font-size: 16px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: opacity 0.2s;
+}
+.btn:hover { opacity: 0.85; }
+.btn-primary { background: #4a6cf7; color: #fff; }
+.btn-danger { background: #f74a4a; color: #fff; }
+
+.mode-display {
+  text-align: center;
+  font-size: 13px;
+  color: #888;
+  margin-bottom: 16px;
+}
+
+.transcript {
+  background: #16213e;
+  border-radius: 8px;
+  padding: 16px;
+  min-height: 200px;
+  max-height: 400px;
+  overflow-y: auto;
+  margin-bottom: 16px;
+}
+.transcript .hint { color: #666; font-style: italic; }
+
+.mode-buttons {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 8px;
+}
+.mode-btn {
+  padding: 10px;
+  background: #16213e;
+  border: 1px solid #333;
+  border-radius: 6px;
+  color: #ccc;
+  font-size: 13px;
+  cursor: pointer;
+  transition: background 0.2s;
+}
+.mode-btn:hover { background: #1a2744; border-color: #4a6cf7; }