commit 59856bd4fcc7da27a85e64cd4377037404031b01 Author: jeremy Date: Wed Apr 1 01:15:24 2026 -0500 feat: initial commit — VoiceCoach real-time voice coaching pipeline Services: Pipecat agent (LiveKit + Claude + Whisper STT + Piper TTS), SpeechBrain analysis service, Piper TTS service, web client (nginx). Includes LiveKit server config and docker-compose orchestration. Co-Authored-By: Claude Sonnet 4.6 diff --git a/.env.example b/.env.example new file mode 100755 index 0000000..4b100ba --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# VoiceCoach Environment Variables +# Copy to .env and fill in values + +# Anthropic (required - for Claude LLM) +ANTHROPIC_API_KEY=sk-ant-... + +# LiveKit (auto-generated on first deploy if not set) +LIVEKIT_API_KEY=voicecoach-key +LIVEKIT_API_SECRET=your-livekit-secret-here diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8fd2ef9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# Environment secrets - never commit +.env + +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +*.egg-info/ +dist/ +build/ +*.egg +.eggs/ +*.whl + +# Virtual environments +venv/ +.venv/ +env/ +ENV/ + +# Model weights and large binaries +*.pt +*.pth +*.onnx +*.bin +*.safetensors +models/ + +# Piper voice models +*.onnx.json + +# Logs +*.log +logs/ + +# Docker +.docker/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store + +# Test artifacts +.pytest_cache/ +.coverage +htmlcov/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100755 index 0000000..483d45f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,127 @@ +services: + + # LiveKit WebRTC Server + livekit-server: + image: livekit/livekit-server:latest + container_name: voicecoach-livekit + restart: unless-stopped + command: + - --config + - /etc/livekit/livekit.yaml + - --keys + - "${LIVEKIT_API_KEY}: ${LIVEKIT_API_SECRET}" + ports: + - "7880:7880" + - "7881:7881" + - "50000-50100:50000-50100/udp" + volumes: + - ./livekit/livekit.yaml:/etc/livekit/livekit.yaml:ro + networks: + - voicecoach + mem_limit: 512M + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # SpeechBrain Emotion Analysis (internal only) + speechbrain-service: + build: + context: ./speechbrain-service + image: voicecoach-speechbrain:latest + container_name: voicecoach-speechbrain + restart: unless-stopped + volumes: + - /mnt/docker-data/configs/voicecoach/speechbrain-models:/app/models + environment: + - MODEL_DIR=/app/models + networks: + - voicecoach + mem_limit: 3G + memswap_limit: 3G + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Piper TTS (internal only) + piper-tts: + build: + context: ./piper-service + image: voicecoach-piper:latest + container_name: voicecoach-piper + restart: unless-stopped + environment: + - PIPER_MODEL=/app/models/en_US-lessac-medium.onnx + networks: + - voicecoach + mem_limit: 512M + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Pipecat Voice Pipeline Agent + pipecat-agent: + build: + context: ./pipecat-agent + image: voicecoach-pipecat:latest + container_name: voicecoach-pipecat + restart: unless-stopped + env_file: + - .env + environment: + - LIVEKIT_URL=ws://livekit-server:7880 + - SPEECHBRAIN_SERVICE_URL=http://speechbrain-service:8001 + - PIPER_SERVICE_URL=http://piper-tts:8003 + - WHISPER_MODEL=medium + depends_on: + - livekit-server + - speechbrain-service + - piper-tts + networks: + - voicecoach + mem_limit: 3G + memswap_limit: 3G + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Web Client (phone interface) + voicecoach-web: + build: + context: ./web-client + image: voicecoach-web:latest + container_name: voicecoach-web + restart: unless-stopped + ports: + - "8085:80" + depends_on: + - pipecat-agent + networks: + - voicecoach + mem_limit: 128M + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +networks: + voicecoach: + driver: bridge diff --git a/livekit/livekit.yaml b/livekit/livekit.yaml new file mode 100755 index 0000000..f424f75 --- /dev/null +++ b/livekit/livekit.yaml @@ -0,0 +1,15 @@ +port: 7880 +rtc: + tcp_port: 7881 + port_range_start: 50000 + port_range_end: 60000 + use_external_ip: true + +logging: + json: false + level: info + +room: + max_participants: 5 + empty_timeout: 300 + departure_timeout: 30 diff --git a/pipecat-agent/Dockerfile b/pipecat-agent/Dockerfile new file mode 100755 index 0000000..9f4590e --- /dev/null +++ b/pipecat-agent/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download faster-whisper medium model at build time +RUN python -c "\ +from faster_whisper import WhisperModel; \ +WhisperModel('medium', device='cpu', compute_type='int8'); \ +print('Whisper model downloaded.')" + +COPY . . + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ + CMD python -c "print('pipeline process alive')" || exit 1 + +CMD ["python", "main.py"] diff --git a/pipecat-agent/main.py b/pipecat-agent/main.py new file mode 100755 index 0000000..c543a39 --- /dev/null +++ b/pipecat-agent/main.py @@ -0,0 +1,46 @@ +""" +Entry point for the Pipecat VoiceCoach agent. +Reads config from environment variables. +""" +import asyncio +import logging +import os + +import structlog +from livekit import api as livekit_api + +from pipeline import run_voice_coach_pipeline + +logging.basicConfig(level=logging.INFO) +logger = structlog.get_logger() + + +async def main(): + anthropic_key = os.environ["ANTHROPIC_API_KEY"] + livekit_api_key = os.environ["LIVEKIT_API_KEY"] + livekit_api_secret = os.environ["LIVEKIT_API_SECRET"] + livekit_url = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880") + + # Generate LiveKit access token for this agent + token = livekit_api.AccessToken(livekit_api_key, livekit_api_secret) + token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) + token.identity = "voicecoach-agent" + livekit_token = token.to_jwt() + + session_context = { + "baseline_context": "", + "session_history": "", + "current_mode": "coaching mode", + } + + logger.info("Starting VoiceCoach pipeline", livekit_url=livekit_url) + await run_voice_coach_pipeline( + livekit_url=livekit_url, + livekit_token=livekit_token, + anthropic_key=anthropic_key, + session_context=session_context, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pipecat-agent/pipeline.py b/pipecat-agent/pipeline.py new file mode 100755 index 0000000..3426eeb --- /dev/null +++ b/pipecat-agent/pipeline.py @@ -0,0 +1,222 @@ +""" +Main Pipecat pipeline assembly. +Connects LiveKit -> Whisper STT -> SpeechBrain -> Claude -> Piper TTS -> LiveKit. +Fully self-hosted STT/TTS (no external API dependencies except Anthropic). +""" +import asyncio +import io +import os +import logging +import wave +import httpx +from typing import Optional + +from pipecat.frames.frames import ( + AudioRawFrame, TranscriptionFrame, TextFrame, EndFrame, TTSAudioRawFrame +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams +from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.vad.silero import SileroVADAnalyzer + +from prompt_builder import build_system_prompt, build_user_message, VocalState +from whisper_stt import LocalWhisperSTT +from piper_tts import PiperTTSClient + +logger = logging.getLogger(__name__) + +SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_SERVICE_URL", "http://speechbrain-service:8001") +PIPER_URL = os.getenv("PIPER_SERVICE_URL", "http://piper-tts:8003") +ANTHROPIC_MODEL = "claude-sonnet-4-6" +WHISPER_MODEL = os.getenv("WHISPER_MODEL", "medium") + + +class WhisperSTTProcessor(FrameProcessor): + """Buffers audio from VAD and transcribes with faster-whisper on end-of-speech.""" + + def __init__(self, whisper: LocalWhisperSTT, **kwargs): + super().__init__(**kwargs) + self._whisper = whisper + self._audio_buffer: list[bytes] = [] + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, AudioRawFrame): + self._audio_buffer.append(frame.audio) + elif isinstance(frame, EndFrame): + # VAD detected end of speech -- transcribe + if self._audio_buffer: + audio_bytes = b"".join(self._audio_buffer) + self._audio_buffer = [] + text = self._whisper.transcribe(audio_bytes) + if text.strip(): + await self.push_frame( + TranscriptionFrame(text=text, user_id="jeremy", timestamp=""), + direction, + ) + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + +class VocalStateInjector(FrameProcessor): + """Intercepts transcriptions, fetches vocal analysis from SpeechBrain, + and enriches the frame before passing to the LLM.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._current_audio: list[bytes] = [] + self._http_client = httpx.AsyncClient(timeout=3.0) + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, AudioRawFrame): + self._current_audio.append(frame.audio) + elif isinstance(frame, TranscriptionFrame): + vocal_state = await self._analyze_buffered_audio() + self._current_audio = [] + enriched_text = build_user_message(frame.text, vocal_state) + enriched_frame = TranscriptionFrame( + text=enriched_text, user_id=frame.user_id, timestamp=frame.timestamp, + ) + await self.push_frame(enriched_frame, direction) + else: + await self.push_frame(frame, direction) + + async def _analyze_buffered_audio(self) -> Optional[VocalState]: + if not self._current_audio: + return None + try: + audio_bytes = b"".join(self._current_audio) + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + wf.writeframes(audio_bytes) + wav_bytes = buf.getvalue() + + response = await self._http_client.post( + f"{SPEECHBRAIN_URL}/analyze", + files={"audio": ("utterance.wav", wav_bytes, "audio/wav")}, + ) + response.raise_for_status() + data = response.json() + return VocalState( + emotion=data.get("emotion", "neutral"), + confidence=data.get("emotion_confidence", 0.0), + pitch_mean_hz=data.get("pitch_mean_hz"), + energy_rms=data.get("energy_rms", 0.0), + speaking_rate_approx=data.get("speaking_rate_approx", 0.0), + voiced_fraction=data.get("voiced_fraction", 0.0), + ) + except Exception as e: + logger.warning(f"SpeechBrain analysis failed (continuing without): {e}") + return None + + +class PiperTTSProcessor(FrameProcessor): + """Converts text frames to audio via Piper TTS service.""" + + def __init__(self, piper_client: PiperTTSClient, **kwargs): + super().__init__(**kwargs) + self._piper = piper_client + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TextFrame): + wav_bytes = await self._piper.synthesize(frame.text) + if wav_bytes: + await self.push_frame( + TTSAudioRawFrame(audio=wav_bytes, sample_rate=22050, num_channels=1), + direction, + ) + else: + await self.push_frame(frame, direction) + + +async def run_voice_coach_pipeline( + livekit_url: str, + livekit_token: str, + anthropic_key: str, + session_context: dict, +): + """Assemble and run the full voice coaching pipeline.""" + + whisper = LocalWhisperSTT(model_size=WHISPER_MODEL, device="cpu", compute_type="int8") + piper_client = PiperTTSClient(base_url=PIPER_URL) + + transport = LiveKitTransport( + url=livekit_url, + token=livekit_token, + room_name="voicecoach", + params=LiveKitParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + whisper_stt = WhisperSTTProcessor(whisper) + vocal_injector = VocalStateInjector() + + llm = AnthropicLLMService( + api_key=anthropic_key, + model=ANTHROPIC_MODEL, + max_tokens=512, + ) + + system_prompt = build_system_prompt( + baseline_context=session_context.get("baseline_context", ""), + session_history=session_context.get("session_history", ""), + current_mode=session_context.get("current_mode", "coaching mode"), + ) + + context = OpenAILLMContext( + messages=[{"role": "system", "content": system_prompt}] + ) + context_aggregator = llm.create_context_aggregator(context) + + piper_tts = PiperTTSProcessor(piper_client) + + pipeline = Pipeline([ + transport.input(), + whisper_stt, + vocal_injector, + context_aggregator.user(), + llm, + piper_tts, + transport.output(), + context_aggregator.assistant(), + ]) + + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) + + runner = PipelineRunner() + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info(f"Participant joined: {participant['identity']}") + await task.queue_frames([ + TextFrame("VoiceCoach connected. How are you doing today?") + ]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info(f"Participant left: {participant['identity']}") + await task.queue_frames([EndFrame()]) + + try: + await runner.run(task) + finally: + await piper_client.close() diff --git a/pipecat-agent/piper_tts.py b/pipecat-agent/piper_tts.py new file mode 100755 index 0000000..83493de --- /dev/null +++ b/pipecat-agent/piper_tts.py @@ -0,0 +1,34 @@ +""" +Piper TTS client for Pipecat pipeline. +Calls the self-hosted Piper HTTP service. +""" +import io +import logging +import wave +import httpx + +logger = logging.getLogger(__name__) + + +class PiperTTSClient: + """HTTP client for the Piper TTS service.""" + + def __init__(self, base_url: str = "http://piper-tts:8003"): + self.base_url = base_url.rstrip("/") + self._client = httpx.AsyncClient(timeout=10.0) + + async def synthesize(self, text: str) -> bytes: + """Convert text to WAV audio bytes via Piper service.""" + try: + response = await self._client.post( + f"{self.base_url}/synthesize", + json={"text": text, "length_scale": 0.95, "sentence_silence": 0.15}, + ) + response.raise_for_status() + return response.content + except Exception as e: + logger.error(f"Piper TTS failed: {e}") + return b"" + + async def close(self): + await self._client.aclose() diff --git a/pipecat-agent/prompt_builder.py b/pipecat-agent/prompt_builder.py new file mode 100755 index 0000000..af7d82f --- /dev/null +++ b/pipecat-agent/prompt_builder.py @@ -0,0 +1,76 @@ +""" +Builds Claude prompts with embedded vocal state metadata. +""" +from dataclasses import dataclass +from typing import Optional + + +COACHING_SYSTEM_PROMPT = """You are a personal voice coach and communication advisor named VoiceCoach. +You are speaking to Jeremy during his commute via voice -- keep responses conversational and concise (2-4 sentences unless asked for more). + +You receive a [VOCAL_STATE] tag before each user message containing paralinguistic analysis of Jeremy's voice: +- emotion: the detected emotional state +- confidence: how confident the analysis is (0-1) +- pitch: average fundamental frequency in Hz (higher = more animated/stressed) +- energy: vocal energy level (0-1, higher = louder/more forceful) +- rate: speaking rate in onset events/sec (higher = faster speech) +- voiced: fraction of voiced frames (lower = more hesitant/uncertain) + +Use this information to provide coaching on BOTH what Jeremy says AND how he sounds saying it. +Important: Never quote the raw numbers. Translate them into natural observations. +Example: "I notice some tension in your voice..." not "Your pitch was 215hz..." + +{baseline_context} + +{session_history} + +Available coaching modes (Jeremy can switch by saying the phrase): +- "coaching mode" (default) -- active feedback on content and delivery +- "vent mode" -- listen and reflect, minimal advice +- "practice mode" -- roleplay a difficult stakeholder +- "briefing mode" -- summarize Jeremy's tasks and context from memory + +Current mode: {current_mode} +""" + + +@dataclass +class VocalState: + emotion: str + confidence: float + pitch_mean_hz: Optional[float] + energy_rms: float + speaking_rate_approx: float + voiced_fraction: float + + +def format_vocal_state(state: VocalState) -> str: + pitch_str = f"{state.pitch_mean_hz:.0f}hz" if state.pitch_mean_hz else "unknown" + return ( + f"[VOCAL_STATE " + f"emotion={state.emotion} " + f"confidence={state.confidence:.2f} " + f"pitch={pitch_str} " + f"energy={state.energy_rms:.2f} " + f"rate={state.speaking_rate_approx:.1f}ops " + f"voiced={state.voiced_fraction:.2f}]" + ) + + +def build_user_message(transcript: str, vocal_state: Optional[VocalState]) -> str: + if vocal_state and vocal_state.confidence > 0.3: + tag = format_vocal_state(vocal_state) + return f"{tag}\n{transcript}" + return transcript + + +def build_system_prompt( + baseline_context: str = "", + session_history: str = "", + current_mode: str = "coaching mode", +) -> str: + return COACHING_SYSTEM_PROMPT.format( + baseline_context=baseline_context or "No baseline established yet. First session: ask Jeremy to read a short neutral calibration script.", + session_history=session_history or "No previous session history.", + current_mode=current_mode, + ) diff --git a/pipecat-agent/requirements.txt b/pipecat-agent/requirements.txt new file mode 100755 index 0000000..37cbc33 --- /dev/null +++ b/pipecat-agent/requirements.txt @@ -0,0 +1,10 @@ +pipecat-ai[livekit,silero,openai]==0.0.48 +anthropic==0.34.0 +faster-whisper==1.0.3 +requests==2.32.3 +httpx==0.27.0 +python-dotenv==1.0.1 +structlog==24.4.0 +numpy==1.26.4 +soundfile==0.12.1 +livekit-api==0.7.1 diff --git a/pipecat-agent/whisper_stt.py b/pipecat-agent/whisper_stt.py new file mode 100755 index 0000000..31f587e --- /dev/null +++ b/pipecat-agent/whisper_stt.py @@ -0,0 +1,35 @@ +""" +Local STT service using faster-whisper. +Replaces Deepgram with fully self-hosted speech-to-text. +""" +import io +import logging +import wave +import numpy as np +from faster_whisper import WhisperModel + +logger = logging.getLogger(__name__) + + +class LocalWhisperSTT: + """Wraps faster-whisper for use in the Pipecat pipeline.""" + + def __init__(self, model_size: str = "medium", device: str = "cpu", compute_type: str = "int8"): + logger.info(f"Loading faster-whisper model: {model_size} on {device}") + self.model = WhisperModel(model_size, device=device, compute_type=compute_type) + logger.info("faster-whisper model loaded.") + + def transcribe(self, audio_bytes: bytes, sample_rate: int = 16000) -> str: + """Transcribe raw PCM audio bytes to text.""" + audio_array = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0 + + segments, info = self.model.transcribe( + audio_array, + beam_size=5, + language="en", + vad_filter=True, + vad_parameters=dict(min_silence_duration_ms=500), + ) + + text = " ".join(seg.text.strip() for seg in segments) + return text diff --git a/piper-service/Dockerfile b/piper-service/Dockerfile new file mode 100755 index 0000000..5661aa1 --- /dev/null +++ b/piper-service/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Download Piper voice model at build time from HuggingFace (lessac medium - good quality, fast on CPU) +RUN mkdir -p /app/models && \ + wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx?download=true" \ + -O /app/models/en_US-lessac-medium.onnx && \ + wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json?download=true" \ + -O /app/models/en_US-lessac-medium.onnx.json + +COPY . . + +EXPOSE 8003 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ + CMD python -c "import httpx; httpx.get('http://localhost:8003/health').raise_for_status()" + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8003", "--workers", "1"] diff --git a/piper-service/main.py b/piper-service/main.py new file mode 100755 index 0000000..a852be2 --- /dev/null +++ b/piper-service/main.py @@ -0,0 +1,72 @@ +""" +Piper TTS HTTP service. +Self-hosted text-to-speech, internal network only. +""" +import io +import logging +import os +import wave +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response +from pydantic import BaseModel +import piper + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_PATH = os.getenv("PIPER_MODEL", "/app/models/en_US-lessac-medium.onnx") +voice = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global voice + logger.info(f"Loading Piper model: {MODEL_PATH}") + voice = piper.PiperVoice.load(MODEL_PATH) + logger.info("Piper TTS ready.") + yield + logger.info("Shutting down.") + + +app = FastAPI(title="Piper TTS Service", lifespan=lifespan) + + +class TTSRequest(BaseModel): + text: str + speaker_id: int = 0 + length_scale: float = 1.0 + sentence_silence: float = 0.2 + + +@app.get("/health") +async def health(): + if voice is None: + raise HTTPException(status_code=503, detail="Model not loaded") + return {"status": "ok", "model": os.path.basename(MODEL_PATH)} + + +@app.post("/synthesize") +async def synthesize(req: TTSRequest): + if voice is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + if not req.text.strip(): + raise HTTPException(status_code=400, detail="Empty text") + + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + voice.synthesize( + req.text, + wf, + speaker_id=req.speaker_id, + length_scale=req.length_scale, + sentence_silence=req.sentence_silence, + ) + + return Response( + content=buf.getvalue(), + media_type="audio/wav", + headers={"Content-Disposition": "inline; filename=speech.wav"}, + ) diff --git a/piper-service/requirements.txt b/piper-service/requirements.txt new file mode 100755 index 0000000..fb05cc6 --- /dev/null +++ b/piper-service/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.0 +piper-tts==1.2.0 +httpx==0.27.0 +numpy==1.26.4 diff --git a/speechbrain-service/Dockerfile b/speechbrain-service/Dockerfile new file mode 100755 index 0000000..bfd7403 --- /dev/null +++ b/speechbrain-service/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + ffmpeg \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download the SpeechBrain model at build time +RUN python -c "\ +from speechbrain.inference.interfaces import foreign_class; \ +foreign_class( \ + source='speechbrain/emotion-recognition-wav2vec2-IEMOCAP', \ + pymodule_file='custom_interface.py', \ + classname='CustomEncoderWav2vec2Classifier', \ + savedir='/app/models', \ + run_opts={'device': 'cpu'}, \ +); \ +print('Model downloaded successfully.')" + +COPY . . + +EXPOSE 8001 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=90s \ + CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()" + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"] diff --git a/speechbrain-service/analyzer.py b/speechbrain-service/analyzer.py new file mode 100755 index 0000000..42d166a --- /dev/null +++ b/speechbrain-service/analyzer.py @@ -0,0 +1,119 @@ +""" +SpeechBrain + librosa vocal analysis. +Performs emotion classification and acoustic feature extraction from audio. +""" +import io +import logging +import numpy as np +import soundfile as sf +import librosa +from speechbrain.inference.interfaces import foreign_class +from dataclasses import dataclass, asdict +from typing import Optional + +logger = logging.getLogger(__name__) + +SAMPLE_RATE = 16000 +MIN_DURATION_SECONDS = 0.5 + + +@dataclass +class VocalAnalysis: + emotion: str + emotion_confidence: float + emotion_scores: dict + pitch_mean_hz: Optional[float] + pitch_variance: Optional[float] + energy_rms: float + speaking_rate_approx: float + voiced_fraction: float + duration_seconds: float + analysis_error: Optional[str] = None + + +class VocalAnalyzer: + """Loads models once at startup and provides analysis per audio chunk.""" + + def __init__(self, model_dir: str = "/app/models"): + logger.info("Loading SpeechBrain emotion model...") + self.classifier = foreign_class( + source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", + pymodule_file="custom_interface.py", + classname="CustomEncoderWav2vec2Classifier", + savedir=model_dir, + run_opts={"device": "cpu"}, + ) + logger.info("SpeechBrain model loaded successfully.") + + def analyze(self, audio_bytes: bytes, content_type: str = "audio/wav") -> VocalAnalysis: + try: + audio_buf = io.BytesIO(audio_bytes) + audio, sr = sf.read(audio_buf, dtype="float32") + + if audio.ndim > 1: + audio = audio.mean(axis=1) + + if sr != SAMPLE_RATE: + audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE) + sr = SAMPLE_RATE + + duration = len(audio) / sr + + if duration < MIN_DURATION_SECONDS: + return VocalAnalysis( + emotion="neutral", emotion_confidence=0.0, emotion_scores={}, + pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0, + speaking_rate_approx=0.0, voiced_fraction=0.0, + duration_seconds=duration, + analysis_error=f"Audio too short: {duration:.2f}s < {MIN_DURATION_SECONDS}s minimum", + ) + + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + sf.write(tmp.name, audio, sr) + tmp_path = tmp.name + + try: + out_prob, score, index, text_lab = self.classifier.classify_file(tmp_path) + emotion = text_lab[0] if isinstance(text_lab, list) else text_lab + confidence = float(score[0]) if hasattr(score, "__iter__") else float(score) + labels = ["neu", "hap", "sad", "ang", "fru"] + prob_list = out_prob[0].tolist() if hasattr(out_prob[0], "tolist") else list(out_prob[0]) + emotion_scores = {l: round(p, 4) for l, p in zip(labels, prob_list)} + finally: + os.unlink(tmp_path) + + f0, voiced_flag, _ = librosa.pyin( + audio, fmin=librosa.note_to_hz("C2"), fmax=librosa.note_to_hz("C7"), sr=sr, + ) + + voiced_f0 = f0[voiced_flag] if voiced_flag is not None else np.array([]) + pitch_mean = float(np.mean(voiced_f0)) if len(voiced_f0) > 0 else None + pitch_var = float(np.var(voiced_f0)) if len(voiced_f0) > 1 else None + voiced_fraction = float(np.sum(voiced_flag) / len(voiced_flag)) if voiced_flag is not None else 0.0 + + rms = librosa.feature.rms(y=audio) + energy_rms = float(np.mean(rms)) + + onset_frames = librosa.onset.onset_detect(y=audio, sr=sr) + speaking_rate = len(onset_frames) / duration if duration > 0 else 0.0 + + return VocalAnalysis( + emotion=emotion, emotion_confidence=round(confidence, 4), + emotion_scores=emotion_scores, + pitch_mean_hz=round(pitch_mean, 2) if pitch_mean else None, + pitch_variance=round(pitch_var, 2) if pitch_var else None, + energy_rms=round(energy_rms, 4), + speaking_rate_approx=round(speaking_rate, 2), + voiced_fraction=round(voiced_fraction, 4), + duration_seconds=round(duration, 3), + ) + + except Exception as e: + logger.exception(f"Analysis failed: {e}") + return VocalAnalysis( + emotion="neutral", emotion_confidence=0.0, emotion_scores={}, + pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0, + speaking_rate_approx=0.0, voiced_fraction=0.0, + duration_seconds=0.0, analysis_error=str(e), + ) diff --git a/speechbrain-service/main.py b/speechbrain-service/main.py new file mode 100755 index 0000000..ae224f5 --- /dev/null +++ b/speechbrain-service/main.py @@ -0,0 +1,66 @@ +""" +FastAPI service wrapping VocalAnalyzer. +Internal-only -- not exposed via Traefik. +""" +import logging +import os +from contextlib import asynccontextmanager +from fastapi import FastAPI, UploadFile, File, HTTPException +from fastapi.responses import JSONResponse +from dataclasses import asdict + +from analyzer import VocalAnalyzer + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_DIR = os.getenv("MODEL_DIR", "/app/models") +analyzer: VocalAnalyzer = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global analyzer + logger.info("Starting VocalAnalyzer...") + analyzer = VocalAnalyzer(model_dir=MODEL_DIR) + logger.info("VocalAnalyzer ready.") + yield + logger.info("Shutting down.") + + +app = FastAPI(title="SpeechBrain Vocal Analysis Service", lifespan=lifespan) + + +@app.get("/health") +async def health(): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + return {"status": "ok", "model": "emotion-recognition-wav2vec2-IEMOCAP"} + + +@app.post("/analyze") +async def analyze(audio: UploadFile = File(...)): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + content = await audio.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="Empty audio file") + + result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav") + return JSONResponse(content=asdict(result)) + + +@app.post("/calibrate") +async def calibrate(audio: UploadFile = File(...)): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + content = await audio.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="Empty audio file") + + result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav") + result_dict = asdict(result) + result_dict["is_baseline"] = True + return JSONResponse(content=result_dict) diff --git a/speechbrain-service/requirements.txt b/speechbrain-service/requirements.txt new file mode 100755 index 0000000..b045065 --- /dev/null +++ b/speechbrain-service/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.0 +speechbrain==1.0.0 +huggingface_hub==0.19.4 +transformers==4.40.2 +torch==2.3.0 +torchaudio==2.3.0 +librosa==0.10.2 +soundfile==0.12.1 +numpy==1.26.4 +httpx==0.27.0 +python-multipart==0.0.9 diff --git a/web-client/Dockerfile b/web-client/Dockerfile new file mode 100755 index 0000000..f7a5433 --- /dev/null +++ b/web-client/Dockerfile @@ -0,0 +1,8 @@ +FROM nginx:alpine + +COPY index.html /usr/share/nginx/html/ +COPY app.js /usr/share/nginx/html/ +COPY style.css /usr/share/nginx/html/ +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 diff --git a/web-client/app.js b/web-client/app.js new file mode 100755 index 0000000..f4b36eb --- /dev/null +++ b/web-client/app.js @@ -0,0 +1,77 @@ +const LIVEKIT_URL = "wss://voice.hobbs.farm"; +let room = null; + +async function connect() { + setStatus("connecting", "Connecting..."); + try { + // Request a token from the token service + const response = await fetch("/api/token", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ identity: "jeremy-" + Date.now() }), + }); + if (!response.ok) throw new Error("Failed to get token"); + const { token } = await response.json(); + + room = new LivekitClient.Room({ + audioCaptureDefaults: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + room.on(LivekitClient.RoomEvent.Connected, () => { + setStatus("connected", "Connected"); + document.getElementById("connectBtn").style.display = "none"; + document.getElementById("disconnectBtn").style.display = "block"; + }); + + room.on(LivekitClient.RoomEvent.Disconnected, () => { + setStatus("disconnected", "Disconnected"); + document.getElementById("connectBtn").style.display = "block"; + document.getElementById("disconnectBtn").style.display = "none"; + }); + + room.on(LivekitClient.RoomEvent.TrackSubscribed, (track) => { + if (track.kind === LivekitClient.Track.Kind.Audio) { + const element = track.attach(); + document.body.appendChild(element); + element.play(); + } + }); + + await room.connect(LIVEKIT_URL, token); + await room.localParticipant.setMicrophoneEnabled(true); + } catch (err) { + setStatus("disconnected", "Error: " + err.message); + console.error(err); + } +} + +async function disconnect() { + if (room) { + await room.disconnect(); + room = null; + } +} + +function setMode(mode) { + document.getElementById("modeDisplay").textContent = "Mode: " + mode; + if (room && room.localParticipant) { + const encoder = new TextEncoder(); + room.localParticipant.publishData( + encoder.encode(JSON.stringify({ type: "mode_change", mode })), + { reliable: true } + ); + } +} + +function setStatus(state, text) { + const el = document.getElementById("status"); + el.className = "status " + state; + el.textContent = text; +} + +document.getElementById("connectBtn").addEventListener("click", connect); +document.getElementById("disconnectBtn").addEventListener("click", disconnect); diff --git a/web-client/index.html b/web-client/index.html new file mode 100755 index 0000000..b4116c1 --- /dev/null +++ b/web-client/index.html @@ -0,0 +1,38 @@ + + + + + + VoiceCoach + + + +
+
+

VoiceCoach

+
Disconnected
+
+ +
+ + +
+ +
Mode: coaching
+ +
+

Press "Start Session" to connect your voice.

+
+ +
+ + + + +
+
+ + + + + diff --git a/web-client/nginx.conf b/web-client/nginx.conf new file mode 100755 index 0000000..e15bc6d --- /dev/null +++ b/web-client/nginx.conf @@ -0,0 +1,15 @@ +server { + listen 80; + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://pipecat-agent:8080/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/web-client/style.css b/web-client/style.css new file mode 100755 index 0000000..5d1e200 --- /dev/null +++ b/web-client/style.css @@ -0,0 +1,88 @@ +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: #1a1a2e; + color: #eee; + min-height: 100vh; + display: flex; + justify-content: center; + align-items: center; +} + +.container { + max-width: 420px; + width: 100%; + padding: 24px; +} + +.header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 24px; +} + +h1 { font-size: 24px; font-weight: 600; } + +.status { + padding: 4px 12px; + border-radius: 12px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; +} +.status.disconnected { background: #444; color: #aaa; } +.status.connecting { background: #553; color: #ffa; } +.status.connected { background: #253; color: #5f5; } + +.controls { margin-bottom: 20px; } + +.btn { + width: 100%; + padding: 14px; + border: none; + border-radius: 8px; + font-size: 16px; + font-weight: 600; + cursor: pointer; + transition: opacity 0.2s; +} +.btn:hover { opacity: 0.85; } +.btn-primary { background: #4a6cf7; color: #fff; } +.btn-danger { background: #f74a4a; color: #fff; } + +.mode-display { + text-align: center; + font-size: 13px; + color: #888; + margin-bottom: 16px; +} + +.transcript { + background: #16213e; + border-radius: 8px; + padding: 16px; + min-height: 200px; + max-height: 400px; + overflow-y: auto; + margin-bottom: 16px; +} +.transcript .hint { color: #666; font-style: italic; } + +.mode-buttons { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} +.mode-btn { + padding: 10px; + background: #16213e; + border: 1px solid #333; + border-radius: 6px; + color: #ccc; + font-size: 13px; + cursor: pointer; + transition: background 0.2s; +} +.mode-btn:hover { background: #1a2744; border-color: #4a6cf7; }