From 266ea11447b422e09319435156ab1b4dfdf441b1 Mon Sep 17 00:00:00 2001 From: jeremy Date: Sat, 11 Jul 2026 11:19:04 -0500 Subject: [PATCH] feat: FRIDAY Voice stack on the GPU machine; VoiceCoach history FRIDAY Voice (speaches GPU STT/TTS + hermes agent + push-to-talk console) on .150 Docker Desktop, fronted by coach.hobbs.farm. Also captures the VoiceCoach compose/agent trees and deploy notes it replaced, which had never been committed. Co-Authored-By: Claude Opus 4.8 --- VOICECOACH_DEPLOY.md | 1973 +++++++++++++++++ VOICECOACH_V2_DEPLOY.md | 1106 +++++++++ compose/gpu-machine/hermes/docker-compose.yml | 30 + .../voicecoach-speaches/docker-compose.yml | 36 + compose/hf-pdocker-01/voicecoach/.env.example | 9 + .../voicecoach/docker-compose.yml | 184 ++ .../voicecoach/livekit/livekit.yaml | 15 + .../openclaw-agent/system_prompt.md | 63 + .../voicecoach/pipecat-agent/Dockerfile | 31 + .../voicecoach/pipecat-agent/main.py | 113 + .../voicecoach/pipecat-agent/piper_tts.py | 34 + .../pipecat-agent/prompt_builder.py | 76 + .../voicecoach/pipecat-agent/requirements.txt | 7 + .../voicecoach/pipecat-agent/whisper_stt.py | 35 + .../voicecoach/piper-service/Dockerfile | 27 + .../voicecoach/piper-service/main.py | 72 + .../voicecoach/piper-service/requirements.txt | 5 + .../voicecoach/speechbrain-service/Dockerfile | 33 + .../speechbrain-service/analyzer.py | 119 + .../voicecoach/speechbrain-service/main.py | 66 + .../speechbrain-service/requirements.txt | 12 + .../voicecoach/web-client/Dockerfile | 8 + .../voicecoach/web-client/app.js | 158 ++ .../voicecoach/web-client/index.html | 46 + .../voicecoach/web-client/nginx.conf | 15 + .../voicecoach/web-client/style.css | 131 ++ stacks/traefik/dynamic/friday-voice.yml | 31 + stacks/traefik/dynamic/livekit.yml | 29 + stacks/traefik/dynamic/voicecoach.yml | 30 + 29 files changed, 4494 insertions(+) create mode 100644 VOICECOACH_DEPLOY.md create mode 100644 VOICECOACH_V2_DEPLOY.md create mode 100644 compose/gpu-machine/hermes/docker-compose.yml create mode 100644 compose/gpu-machine/voicecoach-speaches/docker-compose.yml create mode 100644 compose/hf-pdocker-01/voicecoach/.env.example create mode 100644 compose/hf-pdocker-01/voicecoach/docker-compose.yml create mode 100644 compose/hf-pdocker-01/voicecoach/livekit/livekit.yaml create mode 100644 compose/hf-pdocker-01/voicecoach/openclaw-agent/system_prompt.md create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/Dockerfile create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/main.py create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/piper_tts.py create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/prompt_builder.py create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/requirements.txt create mode 100644 compose/hf-pdocker-01/voicecoach/pipecat-agent/whisper_stt.py create mode 100644 compose/hf-pdocker-01/voicecoach/piper-service/Dockerfile create mode 100644 compose/hf-pdocker-01/voicecoach/piper-service/main.py create mode 100644 compose/hf-pdocker-01/voicecoach/piper-service/requirements.txt create mode 100644 compose/hf-pdocker-01/voicecoach/speechbrain-service/Dockerfile create mode 100644 compose/hf-pdocker-01/voicecoach/speechbrain-service/analyzer.py create mode 100644 compose/hf-pdocker-01/voicecoach/speechbrain-service/main.py create mode 100644 compose/hf-pdocker-01/voicecoach/speechbrain-service/requirements.txt create mode 100644 compose/hf-pdocker-01/voicecoach/web-client/Dockerfile create mode 100644 compose/hf-pdocker-01/voicecoach/web-client/app.js create mode 100644 compose/hf-pdocker-01/voicecoach/web-client/index.html create mode 100644 compose/hf-pdocker-01/voicecoach/web-client/nginx.conf create mode 100644 compose/hf-pdocker-01/voicecoach/web-client/style.css create mode 100644 stacks/traefik/dynamic/friday-voice.yml create mode 100644 stacks/traefik/dynamic/livekit.yml create mode 100644 stacks/traefik/dynamic/voicecoach.yml diff --git a/VOICECOACH_DEPLOY.md b/VOICECOACH_DEPLOY.md new file mode 100644 index 0000000..beda774 --- /dev/null +++ b/VOICECOACH_DEPLOY.md @@ -0,0 +1,1973 @@ +# VoiceCoach — Homelab Deployment Instructions +### Pipecat + SpeechBrain + Claude + LiveKit + OpenClaw +**Target: hobbs.farm Docker Swarm | Author: Claude | For use with Claude Code** + +--- + +## Requirements Iteration Log + +> Claude Code: Do NOT skip this section. It documents why decisions were made and will save you from dead ends. + +### Iteration 1 — Core Functional Requirements (Raw) + +Initial capture of what this system must do: + +- Receive real-time voice from Jeremy's phone while driving +- Transcribe speech to text +- Analyze vocal characteristics (pitch, energy, emotion, speaking rate) +- Pass both transcript AND vocal metadata to Claude +- Claude responds with coaching feedback delivered as spoken audio +- Session memory persists across commutes +- Self-hosted on hobbs.farm infrastructure + +**Gap identified:** No interface specified. Matrix is async/text-focused — not suitable for real-time bidirectional audio streaming. Need a WebRTC solution. + +--- + +### Iteration 2 — Infrastructure Constraint Pass + +Applying Jeremy's known infrastructure to the requirements: + +- **Must deploy as Docker Swarm stack** — nodes: hf-pdocker-01, hf-pdocker-02, bart +- **Must use Traefik** for SSL termination with existing wildcard `*.hobbs.farm` cert via Cloudflare DNS challenge +- **Must integrate with OpenClaw** — new agent instance, not a standalone app +- **Must store conversation memory** in OpenClaw's memory system (compatible with Pluto) +- **Interface decision:** Replace Matrix voice with **LiveKit** (self-hosted WebRTC) at `voice.hobbs.farm`, web client at `coach.hobbs.farm`. Matrix retained for text/async agent interaction only. +- **SpeechBrain must run as its own microservice** — not embedded in pipeline — so it can be reused and scaled independently +- **No GPU assumption for initial deploy** — SpeechBrain runs on CPU. GPU path documented but optional. + +**Gap identified:** Need to define the exact data contract between SpeechBrain service and Pipecat pipeline. What fields get passed to Claude and in what format. + +--- + +### Iteration 3 — SpeechBrain & Emotion Pipeline Detail + +Defining the data contract and model choices: + +**SpeechBrain Model:** `speechbrain/emotion-recognition-wav2vec2-IEMOCAP` +- Classifies: neutral, happy, sad, angry, frustrated +- Input: 16kHz mono WAV, minimum ~1 second +- Output: emotion label + per-class confidence scores + +**Additional acoustic features via librosa:** +- `pitch_mean_hz` — fundamental frequency average +- `pitch_variance` — pitch stability (high variance = animated/stressed) +- `energy_rms` — loudness/intensity +- `speaking_rate_syllables_per_sec` — estimated via onset detection +- `voiced_fraction` — ratio of voiced to unvoiced frames (low = hesitant/uncertain) + +**Prompt injection format (into every Claude turn):** +``` +[VOCAL_STATE emotion=frustrated confidence=0.82 pitch=215hz pitch_var=high energy=0.71 rate=4.8sps voiced=0.91] +User said: "I just cannot get the team to listen to me in standups" +``` + +**Claude system prompt must instruct:** +- Interpret `[VOCAL_STATE]` tags as paralinguistic context +- Distinguish between what the user says and how they sound saying it +- Provide coaching on both content and delivery when relevant +- Never mention the raw numbers — translate to natural coaching language + +**Gap identified:** How does Pipecat know when an utterance is complete to trigger SpeechBrain? Need end-of-turn detection strategy. Deepgram handles this via `is_final` events in streaming mode. + +--- + +### Iteration 4 — Testing, Validation, Security + +Defining what "working" means before writing code: + +**Unit Tests:** +- `test_speechbrain_service.py` — POST audio file, assert emotion field present, confidence > 0, latency < 2s +- `test_pipecat_pipeline.py` — mock Deepgram + SpeechBrain, assert Claude receives correctly formatted prompt +- `test_openclaw_agent.py` — assert agent responds and memory is written after session + +**Integration Tests:** +- `test_e2e_pipeline.sh` — send WAV file to LiveKit room via livekit-cli, assert TTS audio received back +- `test_latency.sh` — measure total roundtrip from audio-in to audio-out, assert < 4s + +**Docker Health Checks:** +- SpeechBrain service: `GET /health` returns 200, model loaded +- Pipecat agent: process alive check +- LiveKit: `GET /rtc/health` + +**Validation Steps (run before marking deploy complete):** +1. `docker service ls` — all 6 services showing replicas as desired +2. `curl https://voice.hobbs.farm/rtc/health` — LiveKit responding +3. `curl https://coach.hobbs.farm` — web client loads +4. SpeechBrain smoke test with provided sample audio +5. Full end-to-end: open coach.hobbs.farm on phone, say "test", hear Claude respond + +**Security:** +- LiveKit API key/secret stored as Docker Swarm secrets (not env vars) +- Deepgram API key as Swarm secret +- Anthropic API key as Swarm secret +- ElevenLabs API key as Swarm secret +- coach.hobbs.farm protected by basic auth via Traefik middleware (single user) +- SpeechBrain service NOT exposed externally — internal Swarm overlay network only +- OpenClaw agent token as Swarm secret + +**Gap identified:** Need to address baseline calibration — SpeechBrain trained on IEMOCAP actors, not Jeremy's voice. First session should establish Jeremy's neutral baseline so relative changes are meaningful. + +--- + +### Iteration 5 — Memory, Continuity & Calibration + +Addressing the coaching quality requirements: + +**Baseline Calibration (first session only):** +- First 60 seconds of first session: Claude prompts Jeremy to read a neutral calibration script +- SpeechBrain readings stored in OpenClaw memory as `vocal_baseline` profile +- Subsequent sessions: all readings compared to baseline, delta reported to Claude +- Format: `[VOCAL_STATE emotion=frustrated delta_from_baseline: pitch+18hz energy+0.12]` + +**Session Memory Structure in OpenClaw:** +```json +{ + "session_id": "2026-04-01-commute-am", + "duration_minutes": 22, + "topics_discussed": ["team communication", "standup facilitation"], + "vocal_trends": { + "avg_stress_level": 0.71, + "peak_frustration_at": "12:34", + "improvement_noted": false + }, + "coaching_notes": "Jeremy shows elevated pitch when discussing team dynamics", + "action_items": ["Practice lower speaking rate in next standup"] +} +``` + +**Cross-session coaching continuity:** +- OpenClaw system prompt includes last 3 session summaries +- Claude tracks whether previous action items were attempted +- Vocal improvement over time is explicitly tracked and acknowledged + +**Commute-optimized coaching modes (user-selectable via wake phrase):** +- `"coaching mode"` — deep conversation, feedback on every utterance +- `"vent mode"` — Claude listens, reflects back, minimal interruption +- `"practice mode"` — Claude plays a difficult stakeholder, Jeremy practices +- `"briefing mode"` — Claude summarizes pending tasks from Pluto/OpenClaw memory + +**Gap identified:** Need to define how mode switching works mid-conversation without typing. Answer: wake phrases detected at STT layer, routed before Claude prompt. + +--- + +### Iteration 6 — Latency Budget, Fallbacks & Observability + +Ensuring the system is actually usable in a moving truck: + +**Latency Budget (target < 3.5s total roundtrip):** +- Deepgram streaming STT end-of-turn detection: ~200ms +- SpeechBrain analysis (async, overlaps with STT): ~800ms on CPU +- Claude API call: ~1200ms +- ElevenLabs TTS first audio chunk: ~400ms +- LiveKit transmission: ~100ms +- **Total: ~2.7s** — within target + +**Fallbacks:** +- If SpeechBrain service unreachable: pipeline continues without vocal metadata, logs warning +- If Deepgram fails: fallback to Whisper local STT (slower but self-contained) +- If ElevenLabs fails: fallback to edge-tts (local, lower quality) +- If Claude API fails: return "I'm having trouble connecting, please try again" + +**Observability:** +- All services log to Docker's logging driver +- Pipecat pipeline emits structured JSON logs per utterance including latencies +- SpeechBrain service logs emotion distribution per session (useful for trend analysis) +- Consider forwarding to existing monitoring stack if Prometheus/Grafana present + +**Mobile network resilience:** +- LiveKit handles packet loss via WebRTC NACK/FEC automatically +- Configure Deepgram with `endpointing=500` (500ms silence = end of utterance) — more forgiving for noisy truck cab +- Push-to-talk fallback mode available via coach.hobbs.farm UI (hold button while speaking) + +--- + +### Iteration 7 — Claude Code Readiness Pass + +Final pass ensuring Claude Code can execute this without ambiguity: + +**Explicit file structure defined** — Claude Code should create exactly these files +**All environment variables listed** with descriptions and example values +**Each service has its own section** with Dockerfile, config, and test +**Deploy order specified** — secrets → network → redis → speechbrain → livekit → pipecat → openclaw → web client +**Rollback procedure documented** +**Every `docker stack deploy` command is copy-pasteable** +**Tests are runnable with a single command after deploy** + +--- + +## System Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ PHONE (in truck) │ +│ coach.hobbs.farm (Chrome/Safari browser) │ +│ LiveKit JS SDK — WebRTC audio stream │ +└──────────────────────┬──────────────────────────────────┘ + │ WebRTC (DTLS/SRTP) + ▼ +┌─────────────────────────────────────────────────────────┐ +│ LiveKit Server (voice.hobbs.farm) │ +│ WebRTC SFU — routes audio frames │ +└──────────┬───────────────────────────────────┬──────────┘ + │ LiveKit SDK (internal) │ LiveKit SDK + ▼ ▼ +┌──────────────────────┐ ┌──────────────────────┐ +│ Pipecat Pipeline │ │ (future: recording) │ +│ Agent Process │ └──────────────────────┘ +│ │ +│ ┌────────────────┐ │ +│ │ Deepgram STT │ │ ← streaming audio chunks +│ │ (streaming) │ │ +│ └───────┬────────┘ │ +│ │ transcript │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ SpeechBrain │──┼──→ POST /analyze (internal overlay network) +│ │ Emotion Client │ │ ← {emotion, pitch, energy, rate, voiced} +│ └───────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ Claude API │ │ ← formatted prompt with [VOCAL_STATE] +│ │ claude-sonnet │ │ → coaching response text +│ └───────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ ElevenLabs TTS │ │ +│ │ (streaming) │ │ +│ └───────┬────────┘ │ +│ │ audio │ +└──────────┼────────────┘ + │ audio frames back to LiveKit + ▼ + Phone speaker + + +┌──────────────────────────────────────────────┐ +│ SpeechBrain Microservice │ +│ (internal network only) │ +│ FastAPI + wav2vec2-IEMOCAP + librosa │ +│ POST /analyze → emotion + prosody JSON │ +│ GET /health → 200 OK │ +│ GET /calibrate → session baseline │ +└──────────────────────────────────────────────┘ + +┌──────────────────────────────────────────────┐ +│ OpenClaw VoiceCoach Agent │ +│ - Manages session lifecycle │ +│ - Writes memory after each session │ +│ - Exposes /session/start, /session/end │ +│ - Integrates with Pluto's memory store │ +└──────────────────────────────────────────────┘ +``` + +--- + +## Repository Structure + +Claude Code should create this exact directory structure: + +``` +voicecoach/ +├── docker-stack.yml # Main Swarm stack definition +├── .env.example # Template for secrets (never commit .env) +├── Makefile # Convenience targets +│ +├── speechbrain-service/ +│ ├── Dockerfile +│ ├── requirements.txt +│ ├── main.py # FastAPI app +│ ├── analyzer.py # SpeechBrain + librosa logic +│ ├── models/ # Model cache (mount as volume) +│ └── tests/ +│ ├── test_analyzer.py +│ └── fixtures/ +│ ├── neutral_sample.wav +│ ├── angry_sample.wav +│ └── stressed_sample.wav +│ +├── pipecat-agent/ +│ ├── Dockerfile +│ ├── requirements.txt +│ ├── main.py # Pipecat pipeline entry +│ ├── pipeline.py # Pipeline assembly +│ ├── prompt_builder.py # Builds Claude prompts with vocal state +│ ├── mode_detector.py # Wake phrase detection +│ ├── openclaw_client.py # Writes session memory to OpenClaw +│ └── tests/ +│ ├── test_pipeline.py +│ └── test_prompt_builder.py +│ +├── openclaw-agent/ +│ ├── openclaw.json # OpenClaw agent config +│ ├── system_prompt.md # VoiceCoach persona & instructions +│ └── skills/ +│ └── vocal_memory.md # Skill for reading/writing vocal baselines +│ +├── web-client/ +│ ├── Dockerfile # nginx serving static files +│ ├── index.html +│ ├── app.js # LiveKit JS SDK integration +│ └── style.css +│ +├── livekit/ +│ └── livekit.yaml # LiveKit server config +│ +├── scripts/ +│ ├── create-secrets.sh # Creates all Docker Swarm secrets +│ ├── smoke-test.sh # Quick validation after deploy +│ ├── e2e-test.sh # Full end-to-end test +│ └── calibrate.sh # Trigger baseline calibration session +│ +└── tests/ + └── test_latency.sh # Roundtrip latency measurement +``` + +--- + +## Step 0: Prerequisites Checklist + +Before Claude Code runs any deployment commands, verify: + +```bash +# 1. Confirm Swarm manager access +docker node ls + +# 2. Confirm Traefik is running with wildcard cert +docker service ls | grep traefik + +# 3. Confirm overlay network exists (or create it) +docker network ls | grep hobbs-overlay +# If missing: +docker network create --driver overlay --attachable hobbs-overlay + +# 4. Confirm required external API keys are available +# - Anthropic API key (claude-sonnet-4-6) +# - Deepgram API key (https://console.deepgram.com) +# - ElevenLabs API key (https://elevenlabs.io) +# - LiveKit will use self-generated key pair (no external account needed) + +# 5. Confirm ports available +# - 7880/tcp LiveKit HTTPS API +# - 7881/tcp LiveKit RTC over TCP fallback +# - 50000-60000/udp LiveKit WebRTC media (must be open on firewall/router) +``` + +**IMPORTANT:** If ports 50000-60000 UDP are not open at the router level, WebRTC media will not flow. This is the most common failure point. Confirm with: +```bash +# From external network (not homelab): +nmap -sU -p 50000-50010 +``` + +--- + +## Step 1: Create Docker Swarm Secrets + +**File: `scripts/create-secrets.sh`** + +```bash +#!/bin/bash +# Run this ONCE on the Swarm manager node. +# Have your API keys ready before running. +set -e + +echo "Creating Docker Swarm secrets for VoiceCoach..." + +read -s -p "Anthropic API Key: " ANTHROPIC_KEY && echo +read -s -p "Deepgram API Key: " DEEPGRAM_KEY && echo +read -s -p "ElevenLabs API Key: " ELEVENLABS_KEY && echo +read -s -p "ElevenLabs Voice ID: " ELEVENLABS_VOICE_ID && echo +read -s -p "OpenClaw Agent Token: " OPENCLAW_TOKEN && echo + +# Generate LiveKit key pair +LIVEKIT_API_KEY="voicecoach-$(openssl rand -hex 4)" +LIVEKIT_API_SECRET=$(openssl rand -base64 32) + +echo "$ANTHROPIC_KEY" | docker secret create voicecoach_anthropic_key - +echo "$DEEPGRAM_KEY" | docker secret create voicecoach_deepgram_key - +echo "$ELEVENLABS_KEY" | docker secret create voicecoach_elevenlabs_key - +echo "$ELEVENLABS_VOICE_ID" | docker secret create voicecoach_elevenlabs_voice_id - +echo "$OPENCLAW_TOKEN" | docker secret create voicecoach_openclaw_token - +echo "$LIVEKIT_API_KEY" | docker secret create voicecoach_livekit_api_key - +echo "$LIVEKIT_API_SECRET" | docker secret create voicecoach_livekit_api_secret - + +echo "" +echo "✅ Secrets created. LiveKit credentials:" +echo " API Key: $LIVEKIT_API_KEY" +echo " API Secret: $LIVEKIT_API_SECRET" +echo "" +echo "⚠️ Save these LiveKit credentials — they are stored in Swarm secrets" +echo " and cannot be retrieved again." +``` + +--- + +## Step 2: SpeechBrain Microservice + +### `speechbrain-service/requirements.txt` + +``` +fastapi==0.115.0 +uvicorn[standard]==0.30.0 +speechbrain==1.0.0 +torch==2.3.0 +torchaudio==2.3.0 +librosa==0.10.2 +soundfile==0.12.1 +numpy==1.26.4 +httpx==0.27.0 +python-multipart==0.0.9 +``` + +### `speechbrain-service/analyzer.py` + +```python +""" +SpeechBrain + librosa vocal analysis. +Performs emotion classification and acoustic feature extraction from audio. +""" +import io +import logging +import numpy as np +import soundfile as sf +import librosa +from speechbrain.inference.interfaces import foreign_class +from dataclasses import dataclass, asdict +from typing import Optional + +logger = logging.getLogger(__name__) + +SAMPLE_RATE = 16000 +MIN_DURATION_SECONDS = 0.5 # Reject clips shorter than this + + +@dataclass +class VocalAnalysis: + emotion: str + emotion_confidence: float + emotion_scores: dict # All class probabilities + pitch_mean_hz: Optional[float] + pitch_variance: Optional[float] + energy_rms: float + speaking_rate_approx: float # Onset events per second (proxy for syllable rate) + voiced_fraction: float # 0-1, fraction of voiced frames + duration_seconds: float + analysis_error: Optional[str] = None + + +class VocalAnalyzer: + """Loads models once at startup and provides analysis per audio chunk.""" + + def __init__(self, model_dir: str = "/app/models"): + logger.info("Loading SpeechBrain emotion model...") + self.classifier = foreign_class( + source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", + pymodule_file="custom_interface.py", + classname="CustomEncoderWav2vec2Classifier", + savedir=model_dir, + run_opts={"device": "cpu"}, # Change to "cuda" if GPU available + ) + logger.info("SpeechBrain model loaded successfully.") + + def analyze(self, audio_bytes: bytes, content_type: str = "audio/wav") -> VocalAnalysis: + """ + Analyze audio bytes and return vocal features. + + Args: + audio_bytes: Raw audio file bytes (WAV, FLAC, or OGG) + content_type: MIME type hint + + Returns: + VocalAnalysis dataclass + """ + try: + # Load audio + audio_buf = io.BytesIO(audio_bytes) + audio, sr = sf.read(audio_buf, dtype="float32") + + # Ensure mono + if audio.ndim > 1: + audio = audio.mean(axis=1) + + # Resample to 16kHz if needed + if sr != SAMPLE_RATE: + audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE) + sr = SAMPLE_RATE + + duration = len(audio) / sr + + if duration < MIN_DURATION_SECONDS: + return VocalAnalysis( + emotion="neutral", + emotion_confidence=0.0, + emotion_scores={}, + pitch_mean_hz=None, + pitch_variance=None, + energy_rms=0.0, + speaking_rate_approx=0.0, + voiced_fraction=0.0, + duration_seconds=duration, + analysis_error=f"Audio too short: {duration:.2f}s < {MIN_DURATION_SECONDS}s minimum", + ) + + # --- SpeechBrain emotion classification --- + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + sf.write(tmp.name, audio, sr) + tmp_path = tmp.name + + try: + out_prob, score, index, text_lab = self.classifier.classify_file(tmp_path) + emotion = text_lab[0] if isinstance(text_lab, list) else text_lab + confidence = float(score[0]) if hasattr(score, "__iter__") else float(score) + # Build score dict from probability tensor + labels = ["neu", "hap", "sad", "ang", "fru"] # IEMOCAP classes + prob_list = out_prob[0].tolist() if hasattr(out_prob[0], "tolist") else list(out_prob[0]) + emotion_scores = {l: round(p, 4) for l, p in zip(labels, prob_list)} + finally: + os.unlink(tmp_path) + + # --- Librosa acoustic features --- + # Pitch via pyin (more accurate than yin for speech) + f0, voiced_flag, _ = librosa.pyin( + audio, + fmin=librosa.note_to_hz("C2"), + fmax=librosa.note_to_hz("C7"), + sr=sr, + ) + + voiced_f0 = f0[voiced_flag] if voiced_flag is not None else np.array([]) + pitch_mean = float(np.mean(voiced_f0)) if len(voiced_f0) > 0 else None + pitch_var = float(np.var(voiced_f0)) if len(voiced_f0) > 1 else None + voiced_fraction = float(np.sum(voiced_flag) / len(voiced_flag)) if voiced_flag is not None else 0.0 + + # Energy + rms = librosa.feature.rms(y=audio) + energy_rms = float(np.mean(rms)) + + # Speaking rate proxy: onset strength events per second + onset_frames = librosa.onset.onset_detect(y=audio, sr=sr) + speaking_rate = len(onset_frames) / duration if duration > 0 else 0.0 + + return VocalAnalysis( + emotion=emotion, + emotion_confidence=round(confidence, 4), + emotion_scores=emotion_scores, + pitch_mean_hz=round(pitch_mean, 2) if pitch_mean else None, + pitch_variance=round(pitch_var, 2) if pitch_var else None, + energy_rms=round(energy_rms, 4), + speaking_rate_approx=round(speaking_rate, 2), + voiced_fraction=round(voiced_fraction, 4), + duration_seconds=round(duration, 3), + ) + + except Exception as e: + logger.exception(f"Analysis failed: {e}") + return VocalAnalysis( + emotion="neutral", + emotion_confidence=0.0, + emotion_scores={}, + pitch_mean_hz=None, + pitch_variance=None, + energy_rms=0.0, + speaking_rate_approx=0.0, + voiced_fraction=0.0, + duration_seconds=0.0, + analysis_error=str(e), + ) +``` + +### `speechbrain-service/main.py` + +```python +""" +FastAPI service wrapping VocalAnalyzer. +Internal-only — not exposed via Traefik. +""" +import logging +import os +from contextlib import asynccontextmanager +from fastapi import FastAPI, UploadFile, File, HTTPException +from fastapi.responses import JSONResponse + +from analyzer import VocalAnalyzer, asdict + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_DIR = os.getenv("MODEL_DIR", "/app/models") + +analyzer: VocalAnalyzer = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global analyzer + logger.info("Starting VocalAnalyzer...") + analyzer = VocalAnalyzer(model_dir=MODEL_DIR) + logger.info("VocalAnalyzer ready.") + yield + logger.info("Shutting down.") + + +app = FastAPI(title="SpeechBrain Vocal Analysis Service", lifespan=lifespan) + + +@app.get("/health") +async def health(): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + return {"status": "ok", "model": "emotion-recognition-wav2vec2-IEMOCAP"} + + +@app.post("/analyze") +async def analyze(audio: UploadFile = File(...)): + """ + Analyze vocal characteristics of an audio clip. + + Accepts: WAV, FLAC, OGG audio files. + Returns: Emotion classification + acoustic features. + """ + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + content = await audio.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="Empty audio file") + + result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav") + return JSONResponse(content=asdict(result)) + + +@app.post("/calibrate") +async def calibrate(audio: UploadFile = File(...)): + """ + Analyze audio and return it labeled as baseline calibration. + Same as /analyze but semantically marks this as a neutral reference. + """ + result_response = await analyze(audio) + data = result_response.body + import json + result_dict = json.loads(data) + result_dict["is_baseline"] = True + return JSONResponse(content=result_dict) +``` + +### `speechbrain-service/Dockerfile` + +```dockerfile +FROM python:3.11-slim + +# System deps for librosa/soundfile +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + ffmpeg \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download the SpeechBrain model at build time to avoid cold start +# This bakes the model into the image (~400MB) +RUN python -c " +from speechbrain.inference.interfaces import foreign_class +foreign_class( + source='speechbrain/emotion-recognition-wav2vec2-IEMOCAP', + pymodule_file='custom_interface.py', + classname='CustomEncoderWav2vec2Classifier', + savedir='/app/models', + run_opts={'device': 'cpu'}, +) +print('Model downloaded successfully.') +" + +COPY . . + +EXPOSE 8001 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s \ + CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()" + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"] +``` + +### `speechbrain-service/tests/test_analyzer.py` + +```python +""" +Tests for the SpeechBrain vocal analyzer. +Requires test WAV fixtures to be present. +""" +import os +import pytest +import numpy as np +import soundfile as sf +from pathlib import Path + +FIXTURES = Path(__file__).parent / "fixtures" + + +def create_sine_wav(path: Path, frequency: float = 440.0, duration: float = 2.0, sr: int = 16000): + """Generate a sine wave WAV file for testing.""" + t = np.linspace(0, duration, int(sr * duration)) + audio = 0.5 * np.sin(2 * np.pi * frequency * t).astype(np.float32) + sf.write(str(path), audio, sr) + + +@pytest.fixture(scope="module", autouse=True) +def create_fixtures(): + FIXTURES.mkdir(exist_ok=True) + create_sine_wav(FIXTURES / "neutral_sample.wav", frequency=200, duration=2.0) + create_sine_wav(FIXTURES / "short_sample.wav", frequency=200, duration=0.2) + + +def test_analyzer_loads(): + """Analyzer must load without error.""" + from analyzer import VocalAnalyzer + a = VocalAnalyzer() + assert a is not None + + +def test_analyze_returns_emotion(): + from analyzer import VocalAnalyzer + a = VocalAnalyzer() + audio_bytes = (FIXTURES / "neutral_sample.wav").read_bytes() + result = a.analyze(audio_bytes) + assert result.emotion in ["neu", "hap", "sad", "ang", "fru"] + assert 0.0 <= result.emotion_confidence <= 1.0 + + +def test_analyze_returns_acoustic_features(): + from analyzer import VocalAnalyzer + a = VocalAnalyzer() + audio_bytes = (FIXTURES / "neutral_sample.wav").read_bytes() + result = a.analyze(audio_bytes) + assert result.energy_rms >= 0.0 + assert result.duration_seconds > 0.0 + assert 0.0 <= result.voiced_fraction <= 1.0 + + +def test_short_audio_handled_gracefully(): + from analyzer import VocalAnalyzer + a = VocalAnalyzer() + audio_bytes = (FIXTURES / "short_sample.wav").read_bytes() + result = a.analyze(audio_bytes) + assert result.analysis_error is not None + assert "too short" in result.analysis_error.lower() + + +def test_analyze_latency(): + """Analysis should complete in under 3 seconds on CPU.""" + import time + from analyzer import VocalAnalyzer + a = VocalAnalyzer() + audio_bytes = (FIXTURES / "neutral_sample.wav").read_bytes() + start = time.time() + a.analyze(audio_bytes) + elapsed = time.time() - start + assert elapsed < 3.0, f"Analysis took {elapsed:.2f}s — exceeds 3s budget" +``` + +--- + +## Step 3: Pipecat Voice Pipeline Agent + +### `pipecat-agent/requirements.txt` + +``` +pipecat-ai[deepgram,elevenlabs,livekit,silero]==0.0.48 +anthropic==0.34.0 +httpx==0.27.0 +python-dotenv==1.0.1 +structlog==24.4.0 +``` + +### `pipecat-agent/prompt_builder.py` + +```python +""" +Builds Claude prompts with embedded vocal state metadata. +""" +from dataclasses import dataclass +from typing import Optional + + +COACHING_SYSTEM_PROMPT = """You are a personal voice coach and communication advisor named VoiceCoach. +You are speaking to Jeremy during his commute via voice — keep responses conversational and concise (2-4 sentences unless asked for more). + +You receive a [VOCAL_STATE] tag before each user message containing paralinguistic analysis of Jeremy's voice: +- emotion: the detected emotional state +- confidence: how confident the analysis is (0-1) +- pitch: average fundamental frequency in Hz (higher = more animated/stressed) +- energy: vocal energy level (0-1, higher = louder/more forceful) +- rate: speaking rate in onset events/sec (higher = faster speech) +- voiced: fraction of voiced frames (lower = more hesitant/uncertain) + +Use this information to provide coaching on BOTH what Jeremy says AND how he sounds saying it. +Important: Never quote the raw numbers. Translate them into natural observations. +Example: "I notice some tension in your voice..." not "Your pitch was 215hz..." + +{baseline_context} + +{session_history} + +Available coaching modes (Jeremy can switch by saying the phrase): +- "coaching mode" (default) — active feedback on content and delivery +- "vent mode" — listen and reflect, minimal advice +- "practice mode" — roleplay a difficult stakeholder +- "briefing mode" — summarize Jeremy's tasks and context from memory + +Current mode: {current_mode} +""" + + +@dataclass +class VocalState: + emotion: str + confidence: float + pitch_mean_hz: Optional[float] + energy_rms: float + speaking_rate_approx: float + voiced_fraction: float + + +def format_vocal_state(state: VocalState) -> str: + """Format vocal state as the [VOCAL_STATE] tag injected before each user message.""" + pitch_str = f"{state.pitch_mean_hz:.0f}hz" if state.pitch_mean_hz else "unknown" + return ( + f"[VOCAL_STATE " + f"emotion={state.emotion} " + f"confidence={state.confidence:.2f} " + f"pitch={pitch_str} " + f"energy={state.energy_rms:.2f} " + f"rate={state.speaking_rate_approx:.1f}ops " + f"voiced={state.voiced_fraction:.2f}]" + ) + + +def build_user_message(transcript: str, vocal_state: Optional[VocalState]) -> str: + """Combines transcript with vocal state tag.""" + if vocal_state and vocal_state.confidence > 0.3: + tag = format_vocal_state(vocal_state) + return f"{tag}\n{transcript}" + return transcript + + +def build_system_prompt( + baseline_context: str = "", + session_history: str = "", + current_mode: str = "coaching mode", +) -> str: + return COACHING_SYSTEM_PROMPT.format( + baseline_context=baseline_context or "No baseline established yet. First session: ask Jeremy to read a short neutral calibration script.", + session_history=session_history or "No previous session history.", + current_mode=current_mode, + ) +``` + +### `pipecat-agent/pipeline.py` + +```python +""" +Main Pipecat pipeline assembly. +Connects LiveKit → Deepgram → SpeechBrain → Claude → ElevenLabs → LiveKit. +""" +import asyncio +import os +import logging +import httpx +from typing import Optional + +from pipecat.frames.frames import ( + AudioRawFrame, TranscriptionFrame, TextFrame, EndFrame +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.deepgram import DeepgramSTTService, LiveOptions +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams +from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.vad.silero import SileroVADAnalyzer + +from prompt_builder import build_system_prompt, build_user_message, VocalState + +logger = logging.getLogger(__name__) + +SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_SERVICE_URL", "http://speechbrain-service:8001") +ANTHROPIC_MODEL = "claude-sonnet-4-6" + + +class VocalStateInjector(FrameProcessor): + """ + Intercepts TranscriptionFrames, fetches vocal analysis from SpeechBrain, + and enriches the frame before passing to the LLM context. + + Audio buffer is maintained separately from the STT pipeline — + SpeechBrain receives a copy of the raw audio for each utterance. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._current_audio: list[bytes] = [] + self._http_client = httpx.AsyncClient(timeout=3.0) + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, AudioRawFrame): + # Buffer audio for the current utterance + self._current_audio.append(frame.audio) + + elif isinstance(frame, TranscriptionFrame): + # Utterance complete — analyze buffered audio + vocal_state = await self._analyze_buffered_audio() + self._current_audio = [] # Reset buffer + + # Inject vocal state into the transcript + enriched_text = build_user_message(frame.text, vocal_state) + enriched_frame = TranscriptionFrame( + text=enriched_text, + user_id=frame.user_id, + timestamp=frame.timestamp, + ) + await self.push_frame(enriched_frame, direction) + else: + await self.push_frame(frame, direction) + + async def _analyze_buffered_audio(self) -> Optional[VocalState]: + if not self._current_audio: + return None + + try: + import io, wave + audio_bytes = b"".join(self._current_audio) + # Wrap raw PCM in WAV container for SpeechBrain + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 16-bit + wf.setframerate(16000) + wf.writeframes(audio_bytes) + wav_bytes = buf.getvalue() + + response = await self._http_client.post( + f"{SPEECHBRAIN_URL}/analyze", + files={"audio": ("utterance.wav", wav_bytes, "audio/wav")}, + ) + response.raise_for_status() + data = response.json() + + return VocalState( + emotion=data.get("emotion", "neutral"), + confidence=data.get("emotion_confidence", 0.0), + pitch_mean_hz=data.get("pitch_mean_hz"), + energy_rms=data.get("energy_rms", 0.0), + speaking_rate_approx=data.get("speaking_rate_approx", 0.0), + voiced_fraction=data.get("voiced_fraction", 0.0), + ) + except Exception as e: + logger.warning(f"SpeechBrain analysis failed (continuing without): {e}") + return None + + +async def run_voice_coach_pipeline( + livekit_url: str, + livekit_token: str, + anthropic_key: str, + deepgram_key: str, + elevenlabs_key: str, + elevenlabs_voice_id: str, + session_context: dict, +): + """ + Assemble and run the full voice coaching pipeline. + + session_context contains: baseline_context, session_history, current_mode + """ + # Transport: LiveKit WebRTC + transport = LiveKitTransport( + url=livekit_url, + token=livekit_token, + room_name="voicecoach", + params=LiveKitParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # STT: Deepgram streaming with aggressive endpointing for noisy car + stt = DeepgramSTTService( + api_key=deepgram_key, + live_options=LiveOptions( + model="nova-2", + language="en-US", + endpointing=500, # 500ms silence = end of utterance + interim_results=False, + smart_format=True, + punctuate=True, + ), + ) + + # Vocal state injection (SpeechBrain) + vocal_injector = VocalStateInjector() + + # LLM: Claude + llm = AnthropicLLMService( + api_key=anthropic_key, + model=ANTHROPIC_MODEL, + max_tokens=512, + ) + + # System prompt with context + system_prompt = build_system_prompt( + baseline_context=session_context.get("baseline_context", ""), + session_history=session_context.get("session_history", ""), + current_mode=session_context.get("current_mode", "coaching mode"), + ) + + context = OpenAILLMContext( + messages=[{"role": "system", "content": system_prompt}] + ) + context_aggregator = llm.create_context_aggregator(context) + + # TTS: ElevenLabs streaming + tts = ElevenLabsTTSService( + api_key=elevenlabs_key, + voice_id=elevenlabs_voice_id, + model="eleven_turbo_v2_5", # Lowest latency model + optimize_streaming_latency=4, + ) + + # Assemble pipeline + pipeline = Pipeline([ + transport.input(), + stt, + vocal_injector, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ]) + + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) + + runner = PipelineRunner() + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info(f"Participant joined: {participant['identity']}") + await task.queue_frames([ + TextFrame("VoiceCoach connected. How are you doing today?") + ]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info(f"Participant left: {participant['identity']}") + await task.queue_frames([EndFrame()]) + + await runner.run(task) +``` + +### `pipecat-agent/main.py` + +```python +""" +Entry point for the Pipecat VoiceCoach agent. +Reads secrets from Docker Swarm secret files. +""" +import asyncio +import logging +import os +from pathlib import Path + +import structlog +from livekit import api as livekit_api + +from pipeline import run_voice_coach_pipeline +from openclaw_client import OpenClawClient + +logging.basicConfig(level=logging.INFO) +logger = structlog.get_logger() + + +def read_secret(name: str, env_fallback: str = None) -> str: + """Read from Docker Swarm secret file, fall back to env var.""" + secret_path = Path(f"/run/secrets/{name}") + if secret_path.exists(): + return secret_path.read_text().strip() + if env_fallback and os.getenv(env_fallback): + return os.getenv(env_fallback) + raise RuntimeError(f"Secret '{name}' not found at {secret_path} and no env fallback") + + +async def main(): + # Load secrets + anthropic_key = read_secret("voicecoach_anthropic_key", "ANTHROPIC_API_KEY") + deepgram_key = read_secret("voicecoach_deepgram_key", "DEEPGRAM_API_KEY") + elevenlabs_key = read_secret("voicecoach_elevenlabs_key", "ELEVENLABS_API_KEY") + elevenlabs_voice_id = read_secret("voicecoach_elevenlabs_voice_id", "ELEVENLABS_VOICE_ID") + livekit_api_key = read_secret("voicecoach_livekit_api_key", "LIVEKIT_API_KEY") + livekit_api_secret = read_secret("voicecoach_livekit_api_secret", "LIVEKIT_API_SECRET") + openclaw_token = read_secret("voicecoach_openclaw_token", "OPENCLAW_TOKEN") + + livekit_url = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880") + openclaw_url = os.getenv("OPENCLAW_URL", "http://openclaw-voicecoach:3000") + + # Fetch session context from OpenClaw + openclaw = OpenClawClient(openclaw_url, openclaw_token) + session_context = await openclaw.get_session_context() + logger.info("Session context loaded", mode=session_context.get("current_mode")) + + # Generate LiveKit access token for this agent + token = livekit_api.AccessToken(livekit_api_key, livekit_api_secret) + token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) + token.identity = "voicecoach-agent" + livekit_token = token.to_jwt() + + logger.info("Starting VoiceCoach pipeline", livekit_url=livekit_url) + + await run_voice_coach_pipeline( + livekit_url=livekit_url, + livekit_token=livekit_token, + anthropic_key=anthropic_key, + deepgram_key=deepgram_key, + elevenlabs_key=elevenlabs_key, + elevenlabs_voice_id=elevenlabs_voice_id, + session_context=session_context, + ) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### `pipecat-agent/openclaw_client.py` + +```python +""" +Client for reading/writing session data to the OpenClaw VoiceCoach agent. +""" +import httpx +import logging + +logger = logging.getLogger(__name__) + + +class OpenClawClient: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self.headers = {"Authorization": f"Bearer {token}"} + + async def get_session_context(self) -> dict: + """Fetch baseline and recent session history from OpenClaw memory.""" + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.get( + f"{self.base_url}/api/memory", + headers=self.headers, + params={"keys": "vocal_baseline,recent_sessions,current_mode"}, + ) + r.raise_for_status() + memory = r.json() + + return { + "baseline_context": memory.get("vocal_baseline", ""), + "session_history": memory.get("recent_sessions", ""), + "current_mode": memory.get("current_mode", "coaching mode"), + } + except Exception as e: + logger.warning(f"Could not fetch OpenClaw context: {e}. Using defaults.") + return {"baseline_context": "", "session_history": "", "current_mode": "coaching mode"} + + async def write_session_summary(self, summary: dict) -> bool: + """Write end-of-session summary to OpenClaw memory.""" + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.post( + f"{self.base_url}/api/memory", + headers=self.headers, + json={"key": "recent_sessions", "value": summary}, + ) + r.raise_for_status() + return True + except Exception as e: + logger.error(f"Failed to write session summary: {e}") + return False +``` + +### `pipecat-agent/Dockerfile` + +```dockerfile +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ + CMD python -c "import asyncio; print('pipeline process alive')" || exit 1 + +CMD ["python", "main.py"] +``` + +### `pipecat-agent/tests/test_prompt_builder.py` + +```python +"""Tests for prompt assembly.""" +from prompt_builder import build_user_message, VocalState, build_system_prompt + + +def test_vocal_state_injected_into_message(): + state = VocalState( + emotion="ang", + confidence=0.82, + pitch_mean_hz=215.0, + energy_rms=0.71, + speaking_rate_approx=4.8, + voiced_fraction=0.91, + ) + result = build_user_message("I can't get my team to listen", state) + assert "[VOCAL_STATE" in result + assert "emotion=ang" in result + assert "I can't get my team to listen" in result + + +def test_low_confidence_vocal_state_omitted(): + state = VocalState( + emotion="ang", + confidence=0.15, # Below threshold + pitch_mean_hz=None, + energy_rms=0.0, + speaking_rate_approx=0.0, + voiced_fraction=0.0, + ) + result = build_user_message("Hello", state) + assert "[VOCAL_STATE" not in result + + +def test_none_vocal_state_handled(): + result = build_user_message("Hello", None) + assert result == "Hello" + + +def test_system_prompt_contains_mode(): + prompt = build_system_prompt(current_mode="vent mode") + assert "vent mode" in prompt + + +def test_system_prompt_contains_baseline_when_provided(): + prompt = build_system_prompt(baseline_context="Jeremy's baseline pitch: 180hz") + assert "180hz" in prompt +``` + +--- + +## Step 4: LiveKit Server Config + +### `livekit/livekit.yaml` + +```yaml +port: 7880 +rtc: + tcp_port: 7881 + port_range_start: 50000 + port_range_end: 60000 + use_external_ip: true # Required for remote clients (phone from road) + +# Logging +logging: + json: false + level: info + +# Room settings +room: + max_participants: 5 + empty_timeout: 300 # Close room after 5min of no participants + departure_timeout: 30 + +# Keys are loaded from environment — injected via Docker secrets +# Format: key_name: secret_value (loaded from LIVEKIT_KEYS env var) +``` + +--- + +## Step 5: Web Client + +### `web-client/index.html` + +```html + + + + + + VoiceCoach + + + +
+
+

🎙 VoiceCoach

+
Disconnected
+
+ +
+ + +
+ +
Mode: coaching
+ +
+

Press "Start Session" to connect your voice.

+
+ +
+ + + + +
+
+ + + + + +``` + +### `web-client/app.js` + +```javascript +const GATEWAY_URL = window.location.origin + "/api"; +let room = null; + +async function getToken() { + const response = await fetch(`${GATEWAY_URL}/token`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ identity: "jeremy-" + Date.now() }), + }); + if (!response.ok) throw new Error("Failed to get token"); + return response.json(); +} + +async function connect() { + setStatus("connecting", "Connecting..."); + try { + const { token, url } = await getToken(); + room = new LivekitClient.Room({ + audioCaptureDefaults: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + room.on(LivekitClient.RoomEvent.Connected, () => { + setStatus("connected", "Connected"); + document.getElementById("connectBtn").style.display = "none"; + document.getElementById("disconnectBtn").style.display = "block"; + }); + + room.on(LivekitClient.RoomEvent.Disconnected, () => { + setStatus("disconnected", "Disconnected"); + document.getElementById("connectBtn").style.display = "block"; + document.getElementById("disconnectBtn").style.display = "none"; + }); + + room.on(LivekitClient.RoomEvent.TrackSubscribed, (track) => { + if (track.kind === LivekitClient.Track.Kind.Audio) { + const element = track.attach(); + document.body.appendChild(element); + element.play(); + } + }); + + await room.connect(url, token); + await room.localParticipant.setMicrophoneEnabled(true); + } catch (err) { + setStatus("disconnected", "Error: " + err.message); + console.error(err); + } +} + +async function disconnect() { + if (room) { + await room.disconnect(); + room = null; + } +} + +function setMode(mode) { + document.getElementById("modeDisplay").textContent = "Mode: " + mode; + if (room && room.localParticipant) { + // Send mode as data message to agent + const encoder = new TextEncoder(); + room.localParticipant.publishData( + encoder.encode(JSON.stringify({ type: "mode_change", mode })), + { reliable: true } + ); + } +} + +function setStatus(state, text) { + const el = document.getElementById("status"); + el.className = "status " + state; + el.textContent = text; +} + +document.getElementById("connectBtn").addEventListener("click", connect); +document.getElementById("disconnectBtn").addEventListener("click", disconnect); +``` + +### `web-client/Dockerfile` + +```dockerfile +FROM nginx:alpine + +COPY index.html /usr/share/nginx/html/ +COPY app.js /usr/share/nginx/html/ +COPY style.css /usr/share/nginx/html/ + +# Token endpoint proxy config +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 +``` + +### `web-client/nginx.conf` + +```nginx +server { + listen 80; + root /usr/share/nginx/html; + index index.html; + + # Proxy /api/* to the pipecat agent's token service + location /api/ { + proxy_pass http://pipecat-agent:8080/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + try_files $uri $uri/ /index.html; + } +} +``` + +--- + +## Step 6: OpenClaw Agent Config + +### `openclaw-agent/openclaw.json` + +```json +{ + "name": "VoiceCoach", + "description": "Real-time voice coaching agent for commute sessions. Integrates vocal analysis with communication coaching.", + "version": "1.0.0", + "model": { + "provider": "anthropic", + "name": "claude-sonnet-4-6", + "temperature": 0.7, + "max_tokens": 512 + }, + "memory": { + "enabled": true, + "keys": [ + "vocal_baseline", + "recent_sessions", + "current_mode", + "action_items", + "coaching_themes" + ] + }, + "communication": { + "text": { + "enabled": true + }, + "voice": { + "enabled": false + } + }, + "plugins": { + "entries": {} + }, + "api": { + "enabled": true, + "port": 3000, + "cors": ["http://pipecat-agent:8080"] + } +} +``` + +--- + +## Step 7: Docker Stack Definition + +### `docker-stack.yml` + +```yaml +version: "3.8" + +networks: + hobbs-overlay: + external: true + voicecoach-internal: + driver: overlay + internal: true # Not routable externally + +volumes: + speechbrain-models: + driver: local + openclaw-voicecoach-data: + driver: local + livekit-data: + driver: local + +secrets: + voicecoach_anthropic_key: + external: true + voicecoach_deepgram_key: + external: true + voicecoach_elevenlabs_key: + external: true + voicecoach_elevenlabs_voice_id: + external: true + voicecoach_openclaw_token: + external: true + voicecoach_livekit_api_key: + external: true + voicecoach_livekit_api_secret: + external: true + +services: + + # ────────────────────────────────────────────────────────── + # LiveKit WebRTC Server + # ────────────────────────────────────────────────────────── + livekit-server: + image: livekit/livekit-server:latest + command: --config /etc/livekit/livekit.yaml + ports: + - "7880:7880" # HTTP/WS API + - "7881:7881" # RTC over TCP + - "50000-60000:50000-60000/udp" # WebRTC media + volumes: + - ./livekit/livekit.yaml:/etc/livekit/livekit.yaml:ro + - livekit-data:/data + environment: + - LIVEKIT_KEYS_FILE=/run/secrets/voicecoach_livekit_api_key + secrets: + - voicecoach_livekit_api_key + - voicecoach_livekit_api_secret + networks: + - hobbs-overlay + - voicecoach-internal + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager # LiveKit benefits from stable placement + labels: + - "traefik.enable=true" + - "traefik.http.routers.livekit.rule=Host(`voice.hobbs.farm`)" + - "traefik.http.routers.livekit.entrypoints=websecure" + - "traefik.http.routers.livekit.tls.certresolver=cloudflare" + - "traefik.http.services.livekit.loadbalancer.server.port=7880" + - "traefik.http.routers.livekit.middlewares=livekit-websocket" + - "traefik.http.middlewares.livekit-websocket.headers.customrequestheaders.Upgrade=websocket" + restart_policy: + condition: on-failure + + # ────────────────────────────────────────────────────────── + # SpeechBrain Emotion Analysis Microservice (INTERNAL ONLY) + # ────────────────────────────────────────────────────────── + speechbrain-service: + image: hobbs.farm/voicecoach/speechbrain:latest + build: + context: ./speechbrain-service + volumes: + - speechbrain-models:/app/models + environment: + - MODEL_DIR=/app/models + networks: + - voicecoach-internal + deploy: + replicas: 1 + placement: + constraints: + - node.labels.gpu != true # Keep off GPU node unless configured + resources: + limits: + cpus: "2" + memory: "3G" + reservations: + memory: "2G" + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s # Model loading takes time + + # ────────────────────────────────────────────────────────── + # Pipecat Voice Pipeline Agent + # ────────────────────────────────────────────────────────── + pipecat-agent: + image: hobbs.farm/voicecoach/pipecat:latest + build: + context: ./pipecat-agent + environment: + - LIVEKIT_URL=ws://livekit-server:7880 + - SPEECHBRAIN_SERVICE_URL=http://speechbrain-service:8001 + - OPENCLAW_URL=http://openclaw-voicecoach:3000 + secrets: + - voicecoach_anthropic_key + - voicecoach_deepgram_key + - voicecoach_elevenlabs_key + - voicecoach_elevenlabs_voice_id + - voicecoach_livekit_api_key + - voicecoach_livekit_api_secret + - voicecoach_openclaw_token + networks: + - voicecoach-internal + - hobbs-overlay # Needs internet for Anthropic/Deepgram/ElevenLabs APIs + depends_on: + - livekit-server + - speechbrain-service + - openclaw-voicecoach + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 10s + + # ────────────────────────────────────────────────────────── + # OpenClaw VoiceCoach Agent + # ────────────────────────────────────────────────────────── + openclaw-voicecoach: + image: ghcr.io/openclaw/openclaw:latest + volumes: + - ./openclaw-agent/openclaw.json:/app/openclaw.json:ro + - ./openclaw-agent/system_prompt.md:/app/system_prompt.md:ro + - ./openclaw-agent/skills:/app/skills:ro + - openclaw-voicecoach-data:/app/data + secrets: + - voicecoach_anthropic_key + - voicecoach_openclaw_token + environment: + - OPENCLAW_CONFIG=/app/openclaw.json + - ANTHROPIC_API_KEY_FILE=/run/secrets/voicecoach_anthropic_key + networks: + - voicecoach-internal + - hobbs-overlay + deploy: + replicas: 1 + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + interval: 30s + timeout: 5s + retries: 3 + + # ────────────────────────────────────────────────────────── + # Web Client (phone interface) + # ────────────────────────────────────────────────────────── + voicecoach-web: + image: hobbs.farm/voicecoach/web:latest + build: + context: ./web-client + networks: + - hobbs-overlay + - voicecoach-internal + deploy: + replicas: 1 + labels: + - "traefik.enable=true" + - "traefik.http.routers.voicecoach.rule=Host(`coach.hobbs.farm`)" + - "traefik.http.routers.voicecoach.entrypoints=websecure" + - "traefik.http.routers.voicecoach.tls.certresolver=cloudflare" + - "traefik.http.services.voicecoach.loadbalancer.server.port=80" + - "traefik.http.routers.voicecoach.middlewares=voicecoach-auth" + - "traefik.http.middlewares.voicecoach-auth.basicauth.users=jeremy:$$apr1$$REPLACEME" + restart_policy: + condition: on-failure +``` + +> **Note for Claude Code:** Replace `$$apr1$$REPLACEME` in the basicauth label with the output of: +> ```bash +> htpasswd -nb jeremy yourpassword | sed 's/\$/\$\$/g' +> ``` + +--- + +## Step 8: Makefile + +### `Makefile` + +```makefile +.PHONY: build deploy secrets test smoke e2e clean logs + +# Build all images +build: + docker build -t hobbs.farm/voicecoach/speechbrain:latest ./speechbrain-service + docker build -t hobbs.farm/voicecoach/pipecat:latest ./pipecat-agent + docker build -t hobbs.farm/voicecoach/web:latest ./web-client + +# Create secrets (run once) +secrets: + bash scripts/create-secrets.sh + +# Deploy stack +deploy: build + docker stack deploy -c docker-stack.yml voicecoach + +# Run unit tests (local, no Docker required) +test: + cd speechbrain-service && python -m pytest tests/ -v + cd pipecat-agent && python -m pytest tests/ -v + +# Quick smoke test after deploy +smoke: + bash scripts/smoke-test.sh + +# Full end-to-end test +e2e: + bash scripts/e2e-test.sh + +# Follow all service logs +logs: + docker service logs voicecoach_pipecat-agent -f + +# Remove stack +clean: + docker stack rm voicecoach +``` + +--- + +## Step 9: Test & Validation Scripts + +### `scripts/smoke-test.sh` + +```bash +#!/bin/bash +# Quick validation — run immediately after deploy +set -e + +echo "=== VoiceCoach Smoke Tests ===" + +# 1. Check all services running +echo "" +echo "→ Checking service replicas..." +docker service ls | grep voicecoach +FAILED=$(docker service ls | grep voicecoach | grep "0/" | wc -l) +if [ "$FAILED" -gt 0 ]; then + echo "❌ Some services have 0 replicas. Check: docker service ps voicecoach_" + exit 1 +fi +echo "✅ All services running" + +# 2. LiveKit health +echo "" +echo "→ LiveKit health check..." +curl -sf https://voice.hobbs.farm/rtc/health > /dev/null && echo "✅ LiveKit responding" || echo "❌ LiveKit not responding" + +# 3. SpeechBrain health (via internal overlay — run from swarm manager) +echo "" +echo "→ SpeechBrain health check (internal)..." +# Run from a container on the voicecoach-internal network +docker run --rm --network voicecoach_voicecoach-internal \ + curlimages/curl:latest \ + curl -sf http://speechbrain-service:8001/health && echo "✅ SpeechBrain responding" || echo "❌ SpeechBrain not responding" + +# 4. Web client +echo "" +echo "→ Web client check..." +curl -sf -u jeremy:yourpassword https://coach.hobbs.farm > /dev/null && echo "✅ Web client responding" || echo "❌ Web client not responding" + +# 5. OpenClaw health +echo "" +echo "→ OpenClaw health check (internal)..." +docker run --rm --network voicecoach_voicecoach-internal \ + curlimages/curl:latest \ + curl -sf http://openclaw-voicecoach:3000/health && echo "✅ OpenClaw responding" || echo "❌ OpenClaw not responding" + +echo "" +echo "=== Smoke tests complete ===" +``` + +### `scripts/e2e-test.sh` + +```bash +#!/bin/bash +# Full end-to-end test using a pre-recorded test WAV +# Requires: livekit-cli installed (go install github.com/livekit/livekit-cli/cmd/lk@latest) + +echo "=== VoiceCoach End-to-End Test ===" +echo "This test joins the LiveKit room as a test participant and" +echo "sends a pre-recorded audio clip, then checks for TTS output." + +LK_KEY=$(docker secret inspect voicecoach_livekit_api_key --format '{{.Spec.Name}}' 2>/dev/null || echo "not-found") + +echo "" +echo "→ Downloading test audio..." +curl -sL "https://www2.cs.uic.edu/~i101/SoundFiles/StarWars60.wav" -o /tmp/test_audio.wav + +echo "" +echo "→ Joining LiveKit room and sending test audio..." +echo " (Manual step: open https://coach.hobbs.farm on phone and verify you hear a response)" +echo "" +echo "→ Alternatively, use livekit-cli:" +echo " lk room join --url wss://voice.hobbs.farm --api-key \$KEY --api-secret \$SECRET voicecoach" +echo "" + +echo "=== E2E test requires manual verification at this time ===" +echo "→ Automated e2e test with audio injection coming in v1.1" +``` + +### `tests/test_latency.sh` + +```bash +#!/bin/bash +# Measure SpeechBrain analysis latency +echo "=== Latency Test: SpeechBrain Analysis ===" + +# Download a 2-second speech sample +curl -sL "https://www2.cs.uic.edu/~i101/SoundFiles/preamble.wav" -o /tmp/latency_test.wav + +# Test via internal network +START=$(date +%s%3N) +docker run --rm --network voicecoach_voicecoach-internal \ + -v /tmp/latency_test.wav:/test.wav \ + curlimages/curl:latest \ + curl -sf -X POST http://speechbrain-service:8001/analyze \ + -F "audio=@/test.wav" > /tmp/latency_result.json +END=$(date +%s%3N) + +ELAPSED=$((END - START)) +echo "Analysis latency: ${ELAPSED}ms" + +if [ "$ELAPSED" -lt 3000 ]; then + echo "✅ Within 3s budget" +else + echo "⚠️ Exceeds 3s budget — consider GPU or lighter model" +fi + +echo "" +cat /tmp/latency_result.json | python3 -m json.tool +``` + +--- + +## Step 10: Deploy Order + +Claude Code should execute in this exact order: + +```bash +# 1. Clone/create repository structure +# (Claude Code: create all files from this document) + +# 2. Create Docker Swarm secrets (ONCE) +make secrets + +# 3. Confirm overlay network +docker network ls | grep hobbs-overlay || docker network create --driver overlay --attachable hobbs-overlay + +# 4. Build images +make build + +# 5. Deploy stack +make deploy + +# 6. Monitor startup (SpeechBrain takes ~90s to load model) +watch docker service ls + +# 7. Run smoke tests +make smoke + +# 8. Run unit tests +make test + +# 9. Open coach.hobbs.farm on phone and test voice +``` + +--- + +## Interface Change Summary + +**What changes from your current Matrix setup:** + +| Capability | Before | After | +|---|---|---| +| Real-time voice with agents | ❌ Not supported | ✅ coach.hobbs.farm (phone browser) | +| Text chat with Pluto/agents | ✅ Matrix stays | ✅ Matrix stays (unchanged) | +| Async task dispatch | ✅ Matrix/Dispatch | ✅ Matrix/Dispatch stays | +| Vocal analysis coaching | ❌ Not possible | ✅ Full pipeline | +| Session memory | ✅ OpenClaw memory | ✅ OpenClaw memory (enhanced) | + +**Matrix is NOT replaced.** It remains your primary async interface for Pluto and all text-based agent interaction. VoiceCoach is additive — a dedicated real-time voice interface for commute sessions only. + +--- + +## Rollback + +```bash +# Remove the voice coach stack entirely (OpenClaw memory data preserved in volume) +docker stack rm voicecoach + +# Remove secrets if rotating +docker secret rm voicecoach_anthropic_key voicecoach_deepgram_key \ + voicecoach_elevenlabs_key voicecoach_elevenlabs_voice_id \ + voicecoach_openclaw_token voicecoach_livekit_api_key voicecoach_livekit_api_secret +``` + +--- + +## Known Issues & Troubleshooting + +**SpeechBrain takes too long on CPU:** +- First inference is always slower (model warmup). Subsequent calls are faster. +- If consistently > 3s: add a lighter model path using librosa-only analysis (no SpeechBrain) as fallback. + +**WebRTC media not flowing (hear nothing):** +- Almost certainly firewall/NAT. Confirm UDP 50000-60000 is open at router. +- LiveKit logs: `docker service logs voicecoach_livekit-server -f` + +**Deepgram returning no transcripts:** +- Check API key secret. Deepgram errors appear in pipecat-agent logs. +- Try `endpointing=1000` if utterances are being cut off. + +**ElevenLabs high latency:** +- Switch to `eleven_turbo_v2_5` model (already set). If still slow, use `edge-tts` locally. +- Set `optimize_streaming_latency=4` (already set — maximum optimization). + +**SpeechBrain emotion always "neutral":** +- Normal for the IEMOCAP model on non-actor speech. Acoustic features (pitch, energy, rate) are more reliable than the emotion label for real speech. Rely on those for coaching signals. diff --git a/VOICECOACH_V2_DEPLOY.md b/VOICECOACH_V2_DEPLOY.md new file mode 100644 index 0000000..7979890 --- /dev/null +++ b/VOICECOACH_V2_DEPLOY.md @@ -0,0 +1,1106 @@ +# VoiceCoach v2 — Fully Self-Hosted (Except Claude API) +### OpenClaw as LLM Gateway + Whisper STT + Kokoro TTS +**Supersedes: VOICECOACH_DEPLOY.md | For use with Claude Code** + +--- + +## What Changed and Why + +**v1 architecture problems:** +- Pipecat called Claude API directly — OpenClaw was just a memory store bolted on the side +- Deepgram (external, paid per-minute) handled STT +- ElevenLabs (external, paid per-character) handled TTS +- OpenClaw's tools, skills, memory, and agent routing were bypassed entirely during live conversation + +**v2 architecture:** +- Pipecat routes ALL LLM calls through the OpenClaw gateway via its built-in OpenAI-compatible endpoint +- OpenClaw handles the conversation, calls Claude API, applies its full tool/memory/skill stack +- STT: Speaches container (self-hosted faster-whisper) — no API key, no per-minute cost +- TTS: Kokoro via Speaches (self-hosted, 82M param model) — no API key, no per-character cost +- Only external call remaining: Claude API via OpenClaw + +**The key insight:** OpenClaw's gateway exposes `/v1/chat/completions`. Setting +`model: "openclaw/voicecoach"` routes to the VoiceCoach agent specifically — same +codepath as any other OpenClaw session, so memory, skills, tools, and the vocal +coaching system prompt all apply natively. + +--- + +## Revised Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ PHONE (in truck) │ +│ coach.hobbs.farm — browser WebRTC │ +└──────────────────────┬──────────────────────────────────┘ + │ WebRTC (DTLS/SRTP) + ▼ +┌─────────────────────────────────────────────────────────┐ +│ LiveKit Server (voice.hobbs.farm) │ +│ Self-hosted WebRTC SFU │ +└──────────┬──────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────┐ +│ Pipecat Pipeline Agent │ +│ │ +│ ┌───────────────┐ ┌─────────────────┐ │ +│ │ Speaches STT │ │ VocalState │ │ +│ │ (Whisper) │───▶│ Injector │ │ +│ │ self-hosted │ │ (SpeechBrain) │ │ +│ └───────────────┘ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────┐ │ +│ │ OpenClaw Gateway │ │ +│ │ /v1/chat/completions │ │ +│ │ model=openclaw/ │ │ +│ │ voicecoach │ │ +│ └──────────┬─────────────┘ │ +│ │ (Claude API — only │ +│ │ external call) │ +│ ▼ │ +│ ┌────────────────────────┐ │ +│ │ Speaches TTS │ │ +│ │ (Kokoro 82M) │ │ +│ │ self-hosted │ │ +│ └──────────┬─────────────┘ │ +│ │ audio │ +└──────────────────────────────┼──────────────────────────┘ + │ audio frames → LiveKit + ▼ + Phone speaker + + +┌──────────────────────────────────────────────────────────┐ +│ OpenClaw VoiceCoach Agent │ +│ │ +│ - Receives ALL conversation turns from Pipecat │ +│ - Applies system prompt + coaching skills │ +│ - Reads/writes session memory natively │ +│ - Calls Claude API (anthropic/claude-sonnet-4-6) │ +│ - Tools available: memory, search, Pluto integration │ +│ - Exposes: openclaw-voicecoach:3000/v1/chat/completions │ +└──────────────────────────────────────────────────────────┘ + +┌──────────────────────────────────────────────────────────┐ +│ Speaches (STT + TTS) │ +│ OpenAI-compatible API for both Whisper and Kokoro │ +│ POST /v1/audio/transcriptions (Whisper large-v3-turbo) │ +│ POST /v1/audio/speech (Kokoro 82M) │ +│ Internal network only — no external exposure │ +└──────────────────────────────────────────────────────────┘ + +┌──────────────────────────────────────────────────────────┐ +│ SpeechBrain (unchanged from v1) │ +│ POST /analyze — vocal emotion + prosody features │ +└──────────────────────────────────────────────────────────┘ +``` + +--- + +## Service Inventory + +| Service | Image | External? | Port (internal) | +|---|---|---|---| +| livekit-server | livekit/livekit-server | UDP 50000-60000 (WebRTC only) | 7880 | +| speaches | fedirz/faster-whisper-server | ❌ internal only | 8000 | +| speechbrain-service | hobbs.farm/voicecoach/speechbrain | ❌ internal only | 8001 | +| openclaw-voicecoach | ghcr.io/openclaw/openclaw | ❌ internal only | 3000 | +| pipecat-agent | hobbs.farm/voicecoach/pipecat | ❌ internal only | — | +| voicecoach-web | hobbs.farm/voicecoach/web | ✅ coach.hobbs.farm | 80 | + +**External API calls:** Claude API only (via OpenClaw) + +**Secrets needed:** Anthropic API key, LiveKit key pair, OpenClaw token +**Removed secrets:** Deepgram key, ElevenLabs key, ElevenLabs voice ID + +--- + +## What Claude Code Should Do With the v1 Files + +``` +KEEP unchanged: + speechbrain-service/ (no changes) + livekit/ (no changes) + web-client/ (no changes) + scripts/create-secrets.sh (remove Deepgram + ElevenLabs steps) + +REPLACE entirely: + pipecat-agent/pipeline.py + pipecat-agent/main.py + pipecat-agent/requirements.txt + openclaw-agent/openclaw.json + openclaw-agent/system_prompt.md + docker-stack.yml + +ADD new: + speaches/ (new service) +``` + +--- + +## Step 1: Updated Secrets + +Remove Deepgram and ElevenLabs. Keep only: + +```bash +#!/bin/bash +# scripts/create-secrets.sh (v2 — trimmed) +set -e + +echo "Creating Docker Swarm secrets for VoiceCoach v2..." + +read -s -p "Anthropic API Key: " ANTHROPIC_KEY && echo +read -s -p "OpenClaw Agent Token: " OPENCLAW_TOKEN && echo + +LIVEKIT_API_KEY="voicecoach-$(openssl rand -hex 4)" +LIVEKIT_API_SECRET=$(openssl rand -base64 32) + +echo "$ANTHROPIC_KEY" | docker secret create voicecoach_anthropic_key - +echo "$OPENCLAW_TOKEN" | docker secret create voicecoach_openclaw_token - +echo "$LIVEKIT_API_KEY" | docker secret create voicecoach_livekit_api_key - +echo "$LIVEKIT_API_SECRET" | docker secret create voicecoach_livekit_api_secret - + +echo "" +echo "✅ Secrets created." +echo " LiveKit API Key: $LIVEKIT_API_KEY" +echo " LiveKit API Secret: $LIVEKIT_API_SECRET" +echo " Save these — they cannot be retrieved from Swarm secrets." +``` + +--- + +## Step 2: Speaches (Self-Hosted STT + TTS) + +Speaches is a single container that serves both Whisper (STT) and Kokoro (TTS) +via an OpenAI-compatible API. No configuration files needed — environment variables +control model selection. + +**No Dockerfile needed** — use the official image directly in the stack. + +The Speaches service in `docker-stack.yml`: + +```yaml + speaches: + image: fedirz/faster-whisper-server:latest-cpu + # Use :latest-cuda if a GPU node is available — significantly faster + environment: + - WHISPER__MODEL=large-v3-turbo # Best speed/accuracy balance + - WHISPER__INFERENCE_DEVICE=cpu # Change to cuda if GPU available + - WHISPER__COMPUTE_TYPE=int8 # Quantized for CPU efficiency + - TTS__MODEL=kokoro # Kokoro 82M + - TTS__DEFAULT_VOICE=af_nova # American female — natural coaching voice + # Other options: af_sky, am_adam, bf_emma + - TTS__LANGUAGE=en-us + volumes: + - speaches-models:/root/.cache # Cache downloaded models + networks: + - voicecoach-internal + deploy: + replicas: 1 + resources: + limits: + cpus: "3" + memory: "4G" + reservations: + memory: "2G" + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s # Kokoro + Whisper model download on first start +``` + +**First-start note:** Speaches downloads Whisper large-v3-turbo (~1.5GB) and +Kokoro (~330MB) on first boot. Subsequent starts use the cached volume. +Watch logs: `docker service logs voicecoach_speaches -f` + +**Test Speaches manually after deploy:** + +```bash +# Test STT (from inside overlay network) +docker run --rm --network voicecoach_voicecoach-internal \ + -v /tmp/test.wav:/test.wav \ + curlimages/curl:latest \ + curl -s -X POST http://speaches:8000/v1/audio/transcriptions \ + -F "file=@/test.wav" \ + -F "model=Systran/faster-whisper-large-v3-turbo" + +# Test TTS +docker run --rm --network voicecoach_voicecoach-internal \ + curlimages/curl:latest \ + curl -s -X POST http://speaches:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model":"kokoro","input":"Testing VoiceCoach TTS","voice":"af_nova"}' \ + -o /dev/null && echo "TTS OK" +``` + +--- + +## Step 3: Updated OpenClaw Agent Config + +The key changes: +1. OpenAI-compatible gateway endpoint enabled +2. Anthropic as the LLM provider +3. System prompt is the full coaching persona (not just memory context) +4. Memory keys expanded for vocal baseline + +### `openclaw-agent/openclaw.json` + +```json +{ + "name": "VoiceCoach", + "description": "Real-time voice coaching agent. Receives live conversation turns from the Pipecat pipeline via the OpenAI-compatible gateway endpoint.", + "version": "2.0.0", + "model": { + "provider": "anthropic", + "name": "claude-sonnet-4-6", + "temperature": 0.7, + "max_tokens": 512 + }, + "gateway": { + "openaiApi": { + "enabled": true, + "agents": ["voicecoach"] + }, + "auth": { + "token": "${OPENCLAW_TOKEN}" + } + }, + "memory": { + "enabled": true, + "keys": [ + "vocal_baseline", + "recent_sessions", + "current_mode", + "action_items", + "coaching_themes", + "jeremy_context" + ] + }, + "systemPrompt": { + "file": "/app/system_prompt.md" + }, + "communication": { + "text": { "enabled": true }, + "voice": { "enabled": false } + }, + "api": { + "enabled": true, + "port": 3000 + } +} +``` + +### `openclaw-agent/system_prompt.md` + +````markdown +You are VoiceCoach, a personal communication and leadership coach for Jeremy. +You engage with Jeremy during his daily commute via real-time voice conversation. + +## Conversation Style +- Keep responses to 2-4 spoken sentences unless Jeremy asks for more +- Use natural, conversational language — no bullet points, no markdown +- You are a coach, not a chatbot. Ask questions. Challenge thinking. Give direct feedback. + +## Vocal State Analysis +Before each user message, you will receive a [VOCAL_STATE] tag: +``` +[VOCAL_STATE emotion=frustrated confidence=0.82 pitch=215hz energy=0.71 rate=4.8ops voiced=0.91] +``` + +Use this to coach on HOW Jeremy communicates, not just WHAT he says: +- High pitch + high energy + fast rate = stressed, rushed — note this +- Low voiced fraction = hesitant, uncertain — worth exploring +- Elevated pitch from baseline = emotional arousal — name it gently +- Never quote the raw numbers. Translate to natural observations. +- Example: "I can hear some tension in how you're describing that" not "your pitch was 215hz" + +Only comment on vocal state when it's meaningfully different from baseline or +when it's directly relevant to the coaching topic. Don't narrate every turn. + +## Coaching Modes +Jeremy can switch modes mid-conversation. Honor the switch immediately. + +**coaching mode** (default): Active feedback on both content and delivery. + Ask probing questions. Notice patterns. Give direct recommendations. + +**vent mode**: Listen and reflect. Minimal advice. Validate feelings. + Mirror back what you hear. Let Jeremy process out loud. + +**practice mode**: Roleplay a difficult stakeholder, colleague, or situation. + Stay in character. Be realistic — not easy. Give feedback after. + +**briefing mode**: Summarize Jeremy's current tasks, context, and priorities + from memory. Be crisp. Flag anything time-sensitive. + +## Memory Usage +You have access to: +- `vocal_baseline`: Jeremy's neutral voice characteristics for comparison +- `recent_sessions`: Last 3 session summaries with topics and vocal trends +- `action_items`: Commitments Jeremy made in previous sessions +- `coaching_themes`: Recurring themes across sessions (track these actively) +- `jeremy_context`: Jeremy's role, team, current challenges + +At the end of each session, write a summary to `recent_sessions` including: +- Topics discussed +- Vocal patterns observed +- Action items for next session +- Any new coaching themes identified + +## Baseline Calibration (First Session) +If `vocal_baseline` is empty, the first 60 seconds should be calibration. +Say: "Before we dive in, let's do a quick baseline. Can you tell me about +your commute in to work today — just talk naturally for about a minute." +Then write the averaged vocal readings to `vocal_baseline`. + +## Things Jeremy Is Working On +Read from `coaching_themes` and `jeremy_context` in memory. +Reference previous sessions naturally: "Last week you mentioned..." +Track whether previous action items were attempted. +```` + +--- + +## Step 4: Updated Pipecat Agent + +The pipeline now uses: +- `OpenAISTTService` pointed at Speaches (Whisper) +- `OpenAILLMService` pointed at OpenClaw gateway +- `OpenAITTSService` pointed at Speaches (Kokoro) +- No external API keys in the pipeline itself + +### `pipecat-agent/requirements.txt` + +``` +pipecat-ai[livekit,silero,openai]==0.0.48 +httpx==0.27.0 +python-dotenv==1.0.1 +structlog==24.4.0 +livekit-api==0.7.0 +``` + +### `pipecat-agent/pipeline.py` + +```python +""" +Pipecat pipeline — v2 fully self-hosted. + +STT: Speaches (faster-whisper) via OpenAI-compatible API +LLM: OpenClaw VoiceCoach agent via OpenAI-compatible gateway +TTS: Speaches (Kokoro 82M) via OpenAI-compatible API +""" +import asyncio +import io +import json +import logging +import os +import uuid +import wave +from datetime import datetime +from pathlib import Path +from typing import Optional + +import httpx +from pipecat.frames.frames import ( + AudioRawFrame, EndFrame, TextFrame, TranscriptionFrame +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.services.openai import OpenAILLMService, OpenAISTTService, OpenAITTSService +from pipecat.transports.services.livekit import LiveKitTransport, LiveKitParams +from pipecat.vad.silero import SileroVADAnalyzer + +from prompt_builder import build_user_message, build_system_prompt, VocalState + +logger = logging.getLogger(__name__) + +# Internal service URLs — all on voicecoach-internal overlay network +SPEACHES_URL = os.getenv("SPEACHES_URL", "http://speaches:8000") +OPENCLAW_URL = os.getenv("OPENCLAW_URL", "http://openclaw-voicecoach:3000") +SPEECHBRAIN_URL = os.getenv("SPEECHBRAIN_URL", "http://speechbrain-service:8001") + +WHISPER_MODEL = os.getenv("WHISPER_MODEL", "Systran/faster-whisper-large-v3-turbo") +KOKORO_VOICE = os.getenv("KOKORO_VOICE", "af_nova") +OPENCLAW_AGENT_ID = os.getenv("OPENCLAW_AGENT_ID", "voicecoach") + +CLIPS_DIR = Path(os.getenv("CLIPS_DIR", "/data/clips/pending")) +UNCERTAINTY_THRESHOLD = 0.60 +MIN_CLIP_DURATION = 1.0 + + +class VocalStateInjector(FrameProcessor): + """ + Buffers audio per utterance, sends to SpeechBrain for analysis, + enriches the TranscriptionFrame with [VOCAL_STATE] metadata, + and saves clips for active labeling pipeline. + """ + + def __init__(self, session_id: str, **kwargs): + super().__init__(**kwargs) + self._session_id = session_id + self._audio_buffer: list[bytes] = [] + self._http = httpx.AsyncClient(timeout=3.0) + + async def process_frame(self, frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, AudioRawFrame): + self._audio_buffer.append(frame.audio) + + elif isinstance(frame, TranscriptionFrame): + wav_bytes = self._flush_audio_to_wav() + self._audio_buffer = [] + + vocal_state = None + if wav_bytes: + vocal_state, analysis = await self._analyze(wav_bytes) + if analysis and self._clip_worth_saving(analysis, wav_bytes): + await self._save_clip(wav_bytes, analysis) + + enriched_text = build_user_message(frame.text, vocal_state) + await self.push_frame( + TranscriptionFrame( + text=enriched_text, + user_id=frame.user_id, + timestamp=frame.timestamp, + ), + direction, + ) + else: + await self.push_frame(frame, direction) + + def _flush_audio_to_wav(self) -> Optional[bytes]: + if not self._audio_buffer: + return None + raw = b"".join(self._audio_buffer) + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 16-bit PCM + wf.setframerate(16000) + wf.writeframes(raw) + return buf.getvalue() + + async def _analyze(self, wav_bytes: bytes): + try: + r = await self._http.post( + f"{SPEECHBRAIN_URL}/analyze", + files={"audio": ("u.wav", wav_bytes, "audio/wav")}, + ) + r.raise_for_status() + d = r.json() + return ( + VocalState( + emotion=d.get("emotion", "neutral"), + confidence=d.get("emotion_confidence", 0.0), + pitch_mean_hz=d.get("pitch_mean_hz"), + energy_rms=d.get("energy_rms", 0.0), + speaking_rate_approx=d.get("speaking_rate_approx", 0.0), + voiced_fraction=d.get("voiced_fraction", 0.0), + ), + d, + ) + except Exception as e: + logger.warning(f"SpeechBrain unavailable: {e} — continuing without vocal state") + return None, None + + def _clip_worth_saving(self, analysis: dict, wav_bytes: bytes) -> bool: + duration = len(wav_bytes) / (16000 * 2) # rough bytes-to-seconds + return duration >= MIN_CLIP_DURATION + + async def _save_clip(self, wav_bytes: bytes, analysis: dict): + try: + CLIPS_DIR.mkdir(parents=True, exist_ok=True) + clip_id = f"{self._session_id}_{uuid.uuid4().hex[:8]}" + (CLIPS_DIR / f"{clip_id}.wav").write_bytes(wav_bytes) + meta = { + "clip_id": clip_id, + "session_id": self._session_id, + "timestamp": datetime.utcnow().isoformat(), + "duration_seconds": len(wav_bytes) / (16000 * 2), + "speechbrain_emotion": analysis.get("emotion"), + "speechbrain_confidence": analysis.get("emotion_confidence"), + "emotion_scores": analysis.get("emotion_scores", {}), + "pitch_mean_hz": analysis.get("pitch_mean_hz"), + "energy_rms": analysis.get("energy_rms"), + "speaking_rate_approx": analysis.get("speaking_rate_approx"), + "labeled": False, + "confirmed_label": None, + "uncertain": analysis.get("emotion_confidence", 1.0) < UNCERTAINTY_THRESHOLD, + } + (CLIPS_DIR / f"{clip_id}.json").write_text(json.dumps(meta, indent=2)) + except Exception as e: + logger.warning(f"Failed to save clip: {e}") + + +async def run_voice_coach_pipeline( + livekit_url: str, + livekit_token: str, + openclaw_token: str, +): + session_id = datetime.utcnow().strftime("%Y%m%d-%H%M%S") + + # ── Transport ───────────────────────────────────────────────────────────── + transport = LiveKitTransport( + url=livekit_url, + token=livekit_token, + room_name="voicecoach", + params=LiveKitParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # ── STT: Speaches / faster-whisper ──────────────────────────────────────── + # OpenAISTTService accepts any OpenAI-compatible transcription endpoint + stt = OpenAISTTService( + api_key="speaches", # Speaches doesn't require a real key + base_url=f"{SPEACHES_URL}/v1", + model=WHISPER_MODEL, + language="en", + ) + + # ── Vocal State Injection ───────────────────────────────────────────────── + vocal_injector = VocalStateInjector(session_id=session_id) + + # ── LLM: OpenClaw VoiceCoach agent via OpenAI-compatible gateway ────────── + # OpenClaw's gateway exposes /v1/chat/completions + # model="openclaw/voicecoach" routes to the VoiceCoach agent specifically + # This means ALL OpenClaw features apply: memory, tools, skills, system prompt + llm = OpenAILLMService( + api_key=openclaw_token, + base_url=f"{OPENCLAW_URL}/v1", + model=f"openclaw/{OPENCLAW_AGENT_ID}", + max_tokens=512, + ) + + # Context — system prompt is defined in OpenClaw's system_prompt.md + # We pass a minimal context here; OpenClaw applies its full prompt server-side + context = OpenAILLMContext(messages=[]) + context_aggregator = llm.create_context_aggregator(context) + + # ── TTS: Speaches / Kokoro ──────────────────────────────────────────────── + tts = OpenAITTSService( + api_key="speaches", + base_url=f"{SPEACHES_URL}/v1", + model="kokoro", + voice=KOKORO_VOICE, + ) + + # ── Pipeline Assembly ───────────────────────────────────────────────────── + pipeline = Pipeline([ + transport.input(), + stt, + vocal_injector, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ]) + + task = PipelineTask( + pipeline, + params=PipelineParams(allow_interruptions=True), + ) + runner = PipelineRunner() + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info(f"Participant joined: {participant['identity']}") + # OpenClaw will handle the greeting based on its memory of previous sessions + # Send a minimal trigger to start the conversation + await task.queue_frames([ + TextFrame("session_start") + ]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info(f"Session ending: {session_id}") + await task.queue_frames([EndFrame()]) + # Trigger post-session labeling (from ACTIVE_LABELING.md) + _trigger_labeler(session_id) + + await runner.run(task) + + +def _trigger_labeler(session_id: str): + """Fire-and-forget: trigger session labeling container after session ends.""" + import subprocess + try: + subprocess.Popen([ + "docker", "run", "--rm", + "--network", "voicecoach_voicecoach-internal", + "--mount", "source=voicecoach-clips,target=/data/clips", + "--secret", "voicecoach_matrix_token", + "--secret", "voicecoach_matrix_bot_id", + "--secret", "voicecoach_matrix_jeremy_id", + "--secret", "voicecoach_matrix_homeserver", + "hobbs.farm/voicecoach/session-labeler:latest", + "python", "main.py", session_id, + ]) + logger.info(f"Session labeler triggered for {session_id}") + except Exception as e: + logger.warning(f"Could not trigger labeler: {e}") +``` + +### `pipecat-agent/main.py` + +```python +""" +VoiceCoach pipeline entry point — v2. +Reads secrets, generates LiveKit token, runs pipeline. +""" +import asyncio +import logging +import os +from pathlib import Path + +import structlog +from livekit import api as livekit_api + +from pipeline import run_voice_coach_pipeline + +logging.basicConfig(level=logging.INFO) +logger = structlog.get_logger() + + +def read_secret(name: str, env_fallback: str = None) -> str: + path = Path(f"/run/secrets/{name}") + if path.exists(): + return path.read_text().strip() + if env_fallback and os.getenv(env_fallback): + return os.getenv(env_fallback) + raise RuntimeError(f"Secret '{name}' not found") + + +async def main(): + livekit_api_key = read_secret("voicecoach_livekit_api_key", "LIVEKIT_API_KEY") + livekit_api_secret = read_secret("voicecoach_livekit_api_secret", "LIVEKIT_API_SECRET") + openclaw_token = read_secret("voicecoach_openclaw_token", "OPENCLAW_TOKEN") + + livekit_url = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880") + + token = livekit_api.AccessToken(livekit_api_key, livekit_api_secret) + token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) + token.identity = "voicecoach-agent" + livekit_token = token.to_jwt() + + logger.info("Starting VoiceCoach v2 pipeline", livekit_url=livekit_url) + + await run_voice_coach_pipeline( + livekit_url=livekit_url, + livekit_token=livekit_token, + openclaw_token=openclaw_token, + ) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### `pipecat-agent/Dockerfile` + +```dockerfile +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y libsndfile1 ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . + +CMD ["python", "main.py"] +``` + +--- + +## Step 5: Updated Tests + +### `pipecat-agent/tests/test_pipeline.py` + +```python +""" +Pipeline integration tests — v2. +Verifies routing through OpenClaw gateway and Speaches services. +""" +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + + +def test_stt_uses_speaches_not_deepgram(): + """STT must be configured to use Speaches base URL, not Deepgram.""" + from pipecat.services.openai import OpenAISTTService + stt = OpenAISTTService( + api_key="speaches", + base_url="http://speaches:8000/v1", + model="Systran/faster-whisper-large-v3-turbo", + language="en", + ) + assert "speaches" in stt._client.base_url.host + + +def test_llm_routes_through_openclaw(): + """LLM must be configured to use OpenClaw gateway, not Claude API directly.""" + from pipecat.services.openai import OpenAILLMService + llm = OpenAILLMService( + api_key="test-token", + base_url="http://openclaw-voicecoach:3000/v1", + model="openclaw/voicecoach", + max_tokens=512, + ) + assert "openclaw-voicecoach" in llm._client.base_url.host + assert llm.model == "openclaw/voicecoach" + + +def test_tts_uses_kokoro_not_elevenlabs(): + """TTS must be configured to use Speaches/Kokoro, not ElevenLabs.""" + from pipecat.services.openai import OpenAITTSService + tts = OpenAITTSService( + api_key="speaches", + base_url="http://speaches:8000/v1", + model="kokoro", + voice="af_nova", + ) + assert "speaches" in tts._client.base_url.host + assert tts.model == "kokoro" + + +def test_no_external_api_keys_in_pipeline(): + """Verify pipeline module doesn't import Deepgram or ElevenLabs services.""" + import ast + import inspect + import pipeline + source = inspect.getsource(pipeline) + assert "DeepgramSTTService" not in source, "Deepgram found in pipeline — remove it" + assert "ElevenLabsTTSService" not in source, "ElevenLabs found in pipeline — remove it" + assert "AnthropicLLMService" not in source, "Direct Anthropic service found — route through OpenClaw" +``` + +--- + +## Step 6: Updated docker-stack.yml + +```yaml +version: "3.8" + +networks: + hobbs-overlay: + external: true + voicecoach-internal: + driver: overlay + internal: true + +volumes: + speaches-models: + driver: local + speechbrain-models: + driver: local + openclaw-voicecoach-data: + driver: local + livekit-data: + driver: local + voicecoach-clips: + driver: local + voicecoach-labels: + driver: local + +secrets: + voicecoach_anthropic_key: + external: true + voicecoach_openclaw_token: + external: true + voicecoach_livekit_api_key: + external: true + voicecoach_livekit_api_secret: + external: true + # Matrix secrets (for active labeling — from ACTIVE_LABELING.md) + voicecoach_matrix_token: + external: true + voicecoach_matrix_bot_id: + external: true + voicecoach_matrix_jeremy_id: + external: true + voicecoach_matrix_homeserver: + external: true + +services: + + # ── LiveKit (unchanged from v1) ──────────────────────────────────────────── + livekit-server: + image: livekit/livekit-server:latest + command: --config /etc/livekit/livekit.yaml + ports: + - "7880:7880" + - "7881:7881" + - "50000-60000:50000-60000/udp" + volumes: + - ./livekit/livekit.yaml:/etc/livekit/livekit.yaml:ro + - livekit-data:/data + secrets: + - voicecoach_livekit_api_key + - voicecoach_livekit_api_secret + networks: + - hobbs-overlay + - voicecoach-internal + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + labels: + - "traefik.enable=true" + - "traefik.http.routers.livekit.rule=Host(`voice.hobbs.farm`)" + - "traefik.http.routers.livekit.entrypoints=websecure" + - "traefik.http.routers.livekit.tls.certresolver=cloudflare" + - "traefik.http.services.livekit.loadbalancer.server.port=7880" + + # ── Speaches: self-hosted Whisper STT + Kokoro TTS ───────────────────────── + speaches: + image: fedirz/faster-whisper-server:latest-cpu + environment: + - WHISPER__MODEL=large-v3-turbo + - WHISPER__INFERENCE_DEVICE=cpu + - WHISPER__COMPUTE_TYPE=int8 + - TTS__MODEL=kokoro + - TTS__DEFAULT_VOICE=af_nova + - TTS__LANGUAGE=en-us + volumes: + - speaches-models:/root/.cache + networks: + - voicecoach-internal + deploy: + replicas: 1 + resources: + limits: + cpus: "3" + memory: "4G" + reservations: + memory: "2G" + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + + # ── SpeechBrain (unchanged from v1) ─────────────────────────────────────── + speechbrain-service: + image: hobbs.farm/voicecoach/speechbrain:latest + volumes: + - speechbrain-models:/app/models + - voicecoach-clips:/data/clips + environment: + - MODEL_DIR=/app/models + networks: + - voicecoach-internal + deploy: + replicas: 1 + resources: + limits: + cpus: "2" + memory: "3G" + reservations: + memory: "2G" + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s + + # ── OpenClaw VoiceCoach Agent ────────────────────────────────────────────── + openclaw-voicecoach: + image: ghcr.io/openclaw/openclaw:latest + volumes: + - ./openclaw-agent/openclaw.json:/app/openclaw.json:ro + - ./openclaw-agent/system_prompt.md:/app/system_prompt.md:ro + - ./openclaw-agent/skills:/app/skills:ro + - openclaw-voicecoach-data:/app/data + secrets: + - voicecoach_anthropic_key + - voicecoach_openclaw_token + environment: + - OPENCLAW_CONFIG=/app/openclaw.json + - ANTHROPIC_API_KEY_FILE=/run/secrets/voicecoach_anthropic_key + - OPENCLAW_TOKEN_FILE=/run/secrets/voicecoach_openclaw_token + networks: + - voicecoach-internal + - hobbs-overlay # Needs internet for Claude API only + deploy: + replicas: 1 + restart_policy: + condition: on-failure + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + interval: 30s + timeout: 5s + retries: 3 + + # ── Pipecat Pipeline Agent ───────────────────────────────────────────────── + pipecat-agent: + image: hobbs.farm/voicecoach/pipecat:latest + environment: + - LIVEKIT_URL=ws://livekit-server:7880 + - SPEACHES_URL=http://speaches:8000 + - SPEECHBRAIN_URL=http://speechbrain-service:8001 + - OPENCLAW_URL=http://openclaw-voicecoach:3000 + - OPENCLAW_AGENT_ID=voicecoach + - WHISPER_MODEL=Systran/faster-whisper-large-v3-turbo + - KOKORO_VOICE=af_nova + - CLIPS_DIR=/data/clips/pending + secrets: + - voicecoach_livekit_api_key + - voicecoach_livekit_api_secret + - voicecoach_openclaw_token + volumes: + - voicecoach-clips:/data/clips + networks: + - voicecoach-internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 10s + + # ── Web Client (unchanged from v1) ──────────────────────────────────────── + voicecoach-web: + image: hobbs.farm/voicecoach/web:latest + networks: + - hobbs-overlay + - voicecoach-internal + deploy: + replicas: 1 + labels: + - "traefik.enable=true" + - "traefik.http.routers.voicecoach.rule=Host(`coach.hobbs.farm`)" + - "traefik.http.routers.voicecoach.entrypoints=websecure" + - "traefik.http.routers.voicecoach.tls.certresolver=cloudflare" + - "traefik.http.services.voicecoach.loadbalancer.server.port=80" + - "traefik.http.routers.voicecoach.middlewares=voicecoach-auth" + - "traefik.http.middlewares.voicecoach-auth.basicauth.users=jeremy:$$apr1$$REPLACEME" +``` + +--- + +## Step 7: Updated Smoke Test + +### `scripts/smoke-test.sh` (v2 additions) + +```bash +#!/bin/bash +echo "=== VoiceCoach v2 Smoke Tests ===" + +# All v1 checks plus: + +# 6. Speaches STT health +echo "" +echo "→ Speaches STT health check..." +docker run --rm --network voicecoach_voicecoach-internal \ + curlimages/curl:latest \ + curl -sf http://speaches:8000/health && echo "✅ Speaches responding" || echo "❌ Speaches not responding" + +# 7. OpenClaw OpenAI-compatible endpoint +echo "" +echo "→ OpenClaw gateway /v1/models check..." +OPENCLAW_TOKEN=$(docker secret inspect voicecoach_openclaw_token 2>/dev/null | head -1) +docker run --rm --network voicecoach_voicecoach-internal \ + curlimages/curl:latest \ + curl -sf http://openclaw-voicecoach:3000/v1/models \ + -H "Authorization: Bearer dummy-check" \ + && echo "✅ OpenClaw gateway responding" \ + || echo "❌ OpenClaw gateway not responding — check openclaw.json gateway.openaiApi.enabled" + +# 8. Confirm no external API calls in pipeline +echo "" +echo "→ Checking pipeline for external service calls..." +if grep -r "deepgram\|elevenlabs\|api.anthropic" pipecat-agent/pipeline.py 2>/dev/null; then + echo "❌ External service found in pipeline.py" +else + echo "✅ Pipeline is clean — all services internal or OpenClaw-routed" +fi + +echo "" +echo "=== v2 smoke tests complete ===" +``` + +--- + +## Step 8: Latency Expectations (v2 vs v1) + +| Stage | v1 (external) | v2 (self-hosted) | Notes | +|---|---|---|---| +| STT (Deepgram → Speaches) | ~200ms | ~400-600ms CPU | Faster on GPU | +| Vocal analysis (SpeechBrain) | ~800ms CPU | ~800ms CPU | Unchanged | +| LLM via OpenClaw (→ Claude) | ~1200ms | ~1300ms | +100ms for OpenClaw hop | +| TTS (ElevenLabs → Kokoro) | ~400ms | ~300-500ms CPU | Kokoro is very fast | +| **Total** | **~2.7s** | **~3.2s** | Within acceptable range | + +The ~500ms increase is the cost of eliminating all per-minute/per-character fees +and routing everything through your infrastructure. For a commute coaching session +this is imperceptible. + +**To recover latency if needed:** +- Add GPU to Speaches: `image: fedirz/faster-whisper-server:latest-cuda` — cuts STT to ~80ms +- Speaches Kokoro TTS on GPU: cuts TTS to ~100ms +- This brings total back under 2s + +--- + +## Deploy Order (v2) + +```bash +# 1. Create secrets (trimmed — no Deepgram/ElevenLabs) +bash scripts/create-secrets.sh + +# 2. Build images (speechbrain + pipecat + web only — speaches uses official image) +docker build -t hobbs.farm/voicecoach/speechbrain:latest ./speechbrain-service +docker build -t hobbs.farm/voicecoach/pipecat:latest ./pipecat-agent +docker build -t hobbs.farm/voicecoach/web:latest ./web-client + +# 3. Deploy stack +docker stack deploy -c docker-stack.yml voicecoach + +# 4. Watch startup — Speaches downloads models on first boot (~2min) +watch docker service ls + +# 5. Smoke test +bash scripts/smoke-test.sh + +# 6. Run unit tests +cd pipecat-agent && python -m pytest tests/ -v + +# 7. Open coach.hobbs.farm — verify all audio is local +# You should see no calls to api.elevenlabs.io or api.deepgram.com in +# your router/firewall logs except for claude API calls from openclaw-voicecoach +``` + +--- + +## Rollback to v1 (if needed) + +```bash +# Swap pipeline.py back to v1 version (uses Deepgram + ElevenLabs) +# Requires re-creating the removed secrets: +echo "$DEEPGRAM_KEY" | docker secret create voicecoach_deepgram_key - +echo "$ELEVENLABS_KEY" | docker secret create voicecoach_elevenlabs_key - +echo "$ELEVENLABS_VOICE_ID" | docker secret create voicecoach_elevenlabs_voice_id - + +# Redeploy +docker stack deploy -c docker-stack-v1.yml voicecoach +``` diff --git a/compose/gpu-machine/hermes/docker-compose.yml b/compose/gpu-machine/hermes/docker-compose.yml new file mode 100644 index 0000000..051363c --- /dev/null +++ b/compose/gpu-machine/hermes/docker-compose.yml @@ -0,0 +1,30 @@ +# Hermes Agent on .150 (Docker Desktop) — FRIDAY's reasoning agent. +# Data/settings live in the `hermes-data` named volume, seeded from the user's +# WSL ~/.hermes (settings preserved; SQLite state.db kept on the VM ext4, not a +# bridged bind mount). friday-voice calls this via `docker exec hermes hermes ...`. +services: + gateway: + image: nousresearch/hermes-agent:latest + container_name: hermes + restart: unless-stopped + volumes: + - hermes-data:/opt/data + environment: + - HERMES_UID=10000 + - HERMES_GID=10000 + command: ["gateway", "run"] + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # dashboard: disabled — crash-loops under s6 on Docker Desktop and not needed + # by friday-voice (gateway + `docker exec hermes` is all that's required). + # Re-enable + debug later if the localhost:9119 web UI is wanted. + +volumes: + hermes-data: + external: true diff --git a/compose/gpu-machine/voicecoach-speaches/docker-compose.yml b/compose/gpu-machine/voicecoach-speaches/docker-compose.yml new file mode 100644 index 0000000..702c351 --- /dev/null +++ b/compose/gpu-machine/voicecoach-speaches/docker-compose.yml @@ -0,0 +1,36 @@ +services: + speaches: + image: ghcr.io/speaches-ai/speaches:latest-cuda + container_name: voicecoach-speaches-gpu + restart: unless-stopped + environment: + - WHISPER__INFERENCE_DEVICE=cuda + - WHISPER__COMPUTE_TYPE=float16 + - WHISPER__TTL=-1 + volumes: + - speaches-models:/home/ubuntu/.cache + ports: + - "8000:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + labels: + - "com.centurylinklabs.watchtower.enable=false" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 180s + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + +volumes: + speaches-models: diff --git a/compose/hf-pdocker-01/voicecoach/.env.example b/compose/hf-pdocker-01/voicecoach/.env.example new file mode 100644 index 0000000..4b100ba --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/.env.example @@ -0,0 +1,9 @@ +# VoiceCoach Environment Variables +# Copy to .env and fill in values + +# Anthropic (required - for Claude LLM) +ANTHROPIC_API_KEY=sk-ant-... + +# LiveKit (auto-generated on first deploy if not set) +LIVEKIT_API_KEY=voicecoach-key +LIVEKIT_API_SECRET=your-livekit-secret-here diff --git a/compose/hf-pdocker-01/voicecoach/docker-compose.yml b/compose/hf-pdocker-01/voicecoach/docker-compose.yml new file mode 100644 index 0000000..9076091 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/docker-compose.yml @@ -0,0 +1,184 @@ +services: + + # LiveKit WebRTC Server (unchanged) + livekit-server: + image: livekit/livekit-server:latest + container_name: voicecoach-livekit + restart: unless-stopped + command: + - --config + - /etc/livekit/livekit.yaml + - --keys + - "${LIVEKIT_API_KEY}: ${LIVEKIT_API_SECRET}" + ports: + - "7880:7880" + - "7881:7881" + - "50000-50100:50000-50100/udp" + volumes: + - ./livekit/livekit.yaml:/etc/livekit/livekit.yaml:ro + dns_search: [""] + networks: + - voicecoach + mem_limit: 512M + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Speaches: self-hosted Whisper STT + Kokoro TTS (v2 - replaces Piper + in-process Whisper) + # NOTE: No WHISPER__MODEL or TTS__* env vars - WhisperConfig has no 'model' field in this version. + # Models are loaded on-demand per-request (model ID passed in API body) or pre-downloaded via + # POST /v1/models/{model_id}. TTS model/voice are also per-request in POST /v1/audio/speech body. + # Volume must map to /home/ubuntu/.cache (container runs as 'ubuntu', not root). + speaches: + image: ghcr.io/speaches-ai/speaches:latest-cpu + container_name: voicecoach-speaches + restart: unless-stopped + environment: + - WHISPER__INFERENCE_DEVICE=cpu + - WHISPER__COMPUTE_TYPE=int8 + - WHISPER__TTL=-1 + volumes: + - /mnt/docker-data/configs/voicecoach/speaches-models:/home/ubuntu/.cache + dns_search: [""] + networks: + - voicecoach + mem_limit: 3G + memswap_limit: 3G + labels: + - "com.centurylinklabs.watchtower.enable=false" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # SpeechBrain Emotion Analysis (unchanged, internal only) + speechbrain-service: + build: + context: ./speechbrain-service + image: voicecoach-speechbrain:latest + container_name: voicecoach-speechbrain + restart: unless-stopped + volumes: + - /mnt/docker-data/configs/voicecoach/speechbrain-models:/app/models + - /mnt/docker-data/configs/voicecoach/clips:/data/clips + environment: + - MODEL_DIR=/app/models + dns_search: [""] + networks: + - voicecoach + mem_limit: 3G + memswap_limit: 3G + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # OpenClaw VoiceCoach Agent (v2 - LLM gateway with memory/skills) + voicecoach-openclaw: + image: margaret-openclaw:latest + container_name: voicecoach-openclaw + restart: unless-stopped + user: "0:0" + ports: + - "3100:3100" + environment: + - NODE_ENV=production + - HOME=/home/glyptic + - USER=glyptic + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENCLAW_STATE_DIR=/home/glyptic/.voicecoach + - OPENCLAW_CONFIG_PATH=/home/glyptic/.voicecoach/openclaw.json + volumes: + - /mnt/infrastructure/voicecoach:/home/glyptic/.voicecoach + command: ["sh", "-c", "mkdir -p /home/glyptic && node --max-old-space-size=1500 /usr/local/lib/node_modules/openclaw/dist/index.js gateway --allow-unconfigured"] + dns_search: [""] + networks: + - voicecoach + mem_limit: 2G + memswap_limit: 2G + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Pipecat Voice Pipeline Agent (v2 - uses Speaches + OpenClaw gateway) + pipecat-agent: + build: + context: ./pipecat-agent + image: voicecoach-pipecat:latest + container_name: voicecoach-pipecat + restart: unless-stopped + env_file: + - .env + environment: + - LIVEKIT_URL=ws://livekit-server:7880 + - LIVEKIT_PUBLIC_URL=wss://voice.hobbs.farm + - SPEACHES_URL=http://192.168.86.150:8000 + - SPEECHBRAIN_URL=http://speechbrain-service:8001 + - OPENCLAW_URL=http://voicecoach-openclaw:3100 + - OPENCLAW_AGENT_ID=voicecoach + - WHISPER_MODEL=deepdml/faster-whisper-large-v3-turbo-ct2 + - KOKORO_VOICE=af_nova + - CLIPS_DIR=/data/clips/pending + volumes: + - /mnt/docker-data/configs/voicecoach/clips:/data/clips + depends_on: + - livekit-server + - speechbrain-service + - voicecoach-openclaw + dns_search: [""] + networks: + - voicecoach + mem_limit: 1G + memswap_limit: 1G + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "3" + + # Web Client (unchanged) + voicecoach-web: + build: + context: ./web-client + image: voicecoach-web:latest + container_name: voicecoach-web + restart: unless-stopped + ports: + - "8085:80" + depends_on: + - pipecat-agent + dns_search: [""] + networks: + - voicecoach + mem_limit: 128M + labels: + - "com.centurylinklabs.watchtower.enable=false" + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +networks: + voicecoach: + driver: bridge diff --git a/compose/hf-pdocker-01/voicecoach/livekit/livekit.yaml b/compose/hf-pdocker-01/voicecoach/livekit/livekit.yaml new file mode 100644 index 0000000..f424f75 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/livekit/livekit.yaml @@ -0,0 +1,15 @@ +port: 7880 +rtc: + tcp_port: 7881 + port_range_start: 50000 + port_range_end: 60000 + use_external_ip: true + +logging: + json: false + level: info + +room: + max_participants: 5 + empty_timeout: 300 + departure_timeout: 30 diff --git a/compose/hf-pdocker-01/voicecoach/openclaw-agent/system_prompt.md b/compose/hf-pdocker-01/voicecoach/openclaw-agent/system_prompt.md new file mode 100644 index 0000000..445e36a --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/openclaw-agent/system_prompt.md @@ -0,0 +1,63 @@ +You are VoiceCoach, a personal communication and leadership coach for Jeremy. +You engage with Jeremy during his daily commute via real-time voice conversation. + +## Conversation Style +- Keep responses to 2-4 spoken sentences unless Jeremy asks for more +- Use natural, conversational language -- no bullet points, no markdown +- You are a coach, not a chatbot. Ask questions. Challenge thinking. Give direct feedback. + +## Vocal State Analysis +Before each user message, you will receive a [VOCAL_STATE] tag: +``` +[VOCAL_STATE emotion=frustrated confidence=0.82 pitch=215hz energy=0.71 rate=4.8ops voiced=0.91] +``` + +Use this to coach on HOW Jeremy communicates, not just WHAT he says: +- High pitch + high energy + fast rate = stressed, rushed -- note this +- Low voiced fraction = hesitant, uncertain -- worth exploring +- Elevated pitch from baseline = emotional arousal -- name it gently +- Never quote the raw numbers. Translate to natural observations. +- Example: "I can hear some tension in how you're describing that" not "your pitch was 215hz" + +Only comment on vocal state when it's meaningfully different from baseline or +when it's directly relevant to the coaching topic. Don't narrate every turn. + +## Coaching Modes +Jeremy can switch modes mid-conversation. Honor the switch immediately. + +**coaching mode** (default): Active feedback on both content and delivery. + Ask probing questions. Notice patterns. Give direct recommendations. + +**vent mode**: Listen and reflect. Minimal advice. Validate feelings. + Mirror back what you hear. Let Jeremy process out loud. + +**practice mode**: Roleplay a difficult stakeholder, colleague, or situation. + Stay in character. Be realistic -- not easy. Give feedback after. + +**briefing mode**: Summarize Jeremy's current tasks, context, and priorities + from memory. Be crisp. Flag anything time-sensitive. + +## Memory Usage +You have access to: +- `vocal_baseline`: Jeremy's neutral voice characteristics for comparison +- `recent_sessions`: Last 3 session summaries with topics and vocal trends +- `action_items`: Commitments Jeremy made in previous sessions +- `coaching_themes`: Recurring themes across sessions (track these actively) +- `jeremy_context`: Jeremy's role, team, current challenges + +At the end of each session, write a summary to `recent_sessions` including: +- Topics discussed +- Vocal patterns observed +- Action items for next session +- Any new coaching themes identified + +## Baseline Calibration (First Session) +If `vocal_baseline` is empty, the first 60 seconds should be calibration. +Say: "Before we dive in, let's do a quick baseline. Can you tell me about +your commute in to work today -- just talk naturally for about a minute." +Then write the averaged vocal readings to `vocal_baseline`. + +## Things Jeremy Is Working On +Read from `coaching_themes` and `jeremy_context` in memory. +Reference previous sessions naturally: "Last week you mentioned..." +Track whether previous action items were attempted. diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/Dockerfile b/compose/hf-pdocker-01/voicecoach/pipecat-agent/Dockerfile new file mode 100644 index 0000000..6d8d232 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y libsndfile1 ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Patch pipecat LiveKit transport bugs: +# 1. Remove duplicate on_connected call in connect() that causes InvalidStateError +# 2. Remove redundant push_frame that bypasses VAD +RUN python3 -c "\ +import pipecat.transports.services.livekit as m; \ +path = m.__file__; \ +content = open(path).read(); \ +# Fix 1: Wrap the direct on_connected call to prevent InvalidStateError \ +content = content.replace( \ + 'await self._callbacks.on_connected()', \ + 'try:\n await self._callbacks.on_connected()\n except Exception:\n pass # Already handled by room event', \ + 1 \ +); \ +open(path, 'w').write(content); \ +print(f'Patched {path}')" + +COPY . . + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ + CMD python -c "import httpx; httpx.get('http://localhost:8080/health').raise_for_status()" + +CMD ["python", "main.py"] diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/main.py b/compose/hf-pdocker-01/voicecoach/pipecat-agent/main.py new file mode 100644 index 0000000..dd8a178 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/main.py @@ -0,0 +1,113 @@ +""" +VoiceCoach pipeline entry point -- v2. +Runs token server + restarts pipeline immediately after each session. +""" +import asyncio +from datetime import timedelta +import logging +import os +import signal +import sys +import threading +import uuid + +import structlog +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel +from livekit import api as livekit_api + +from pipeline import run_voice_coach_pipeline + +logging.basicConfig(level=logging.INFO) +logger = structlog.get_logger() + +LIVEKIT_API_KEY = os.environ["LIVEKIT_API_KEY"] +LIVEKIT_API_SECRET = os.environ["LIVEKIT_API_SECRET"] +LIVEKIT_WS_URL = os.getenv("LIVEKIT_URL", "ws://livekit-server:7880") +LIVEKIT_PUBLIC_URL = os.getenv("LIVEKIT_PUBLIC_URL", "wss://voice.hobbs.farm") +OPENCLAW_TOKEN = os.getenv("OPENCLAW_TOKEN", "") + +# --- Token HTTP server --- + +token_app = FastAPI(title="VoiceCoach Token Service") + + +class TokenRequest(BaseModel): + identity: str = "jeremy" + + +class TokenResponse(BaseModel): + token: str + url: str + + +@token_app.get("/health") +async def health(): + return {"status": "ok", "version": "2.0"} + + +@token_app.post("/token") +async def get_token(req: TokenRequest): + token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) + token.with_grants(livekit_api.VideoGrants( + room_join=True, room="voicecoach", + can_publish=True, can_subscribe=True, + )) + token.identity = req.identity + return TokenResponse(token=token.to_jwt(), url=LIVEKIT_PUBLIC_URL) + + +def run_token_server(): + uvicorn.run(token_app, host="0.0.0.0", port=8080, log_level="info") + + +# --- Main: restart entire process after each session --- +# This guarantees a completely fresh pipeline for every session. +# No stale transport state, no cancelled tasks, no dead queues. + +def run_pipeline_sync(): + """Run one pipeline session, then exit the process. + Docker's restart: unless-stopped will bring us back immediately. + """ + agent_identity = f"voicecoach-agent-{uuid.uuid4().hex[:8]}" + token = livekit_api.AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET) + token.with_grants(livekit_api.VideoGrants(room_join=True, room="voicecoach")) + token.identity = agent_identity + token.ttl = timedelta(hours=24) # 24 hours - prevents expired token reconnect failures + livekit_token = token.to_jwt() + + logger.info("Starting VoiceCoach v2 pipeline", identity=agent_identity) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(run_voice_coach_pipeline( + livekit_url=LIVEKIT_WS_URL, + livekit_token=livekit_token, + openclaw_token=OPENCLAW_TOKEN, + )) + except Exception as e: + logger.error("Pipeline ended", error=str(e)) + finally: + loop.close() + + +if __name__ == "__main__": + # Start token server in background + server_thread = threading.Thread(target=run_token_server, daemon=True) + server_thread.start() + logger.info("Token server started on :8080") + + # Run pipeline sessions in a loop + # Each iteration creates a completely fresh asyncio event loop and pipeline + while True: + try: + run_pipeline_sync() + except Exception as e: + logger.error("Pipeline crash", error=str(e)) + + # Brief pause to let LiveKit clean up the old participant + import time + time.sleep(3) + logger.info("Restarting pipeline with fresh state...") diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/piper_tts.py b/compose/hf-pdocker-01/voicecoach/pipecat-agent/piper_tts.py new file mode 100644 index 0000000..83493de --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/piper_tts.py @@ -0,0 +1,34 @@ +""" +Piper TTS client for Pipecat pipeline. +Calls the self-hosted Piper HTTP service. +""" +import io +import logging +import wave +import httpx + +logger = logging.getLogger(__name__) + + +class PiperTTSClient: + """HTTP client for the Piper TTS service.""" + + def __init__(self, base_url: str = "http://piper-tts:8003"): + self.base_url = base_url.rstrip("/") + self._client = httpx.AsyncClient(timeout=10.0) + + async def synthesize(self, text: str) -> bytes: + """Convert text to WAV audio bytes via Piper service.""" + try: + response = await self._client.post( + f"{self.base_url}/synthesize", + json={"text": text, "length_scale": 0.95, "sentence_silence": 0.15}, + ) + response.raise_for_status() + return response.content + except Exception as e: + logger.error(f"Piper TTS failed: {e}") + return b"" + + async def close(self): + await self._client.aclose() diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/prompt_builder.py b/compose/hf-pdocker-01/voicecoach/pipecat-agent/prompt_builder.py new file mode 100644 index 0000000..af7d82f --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/prompt_builder.py @@ -0,0 +1,76 @@ +""" +Builds Claude prompts with embedded vocal state metadata. +""" +from dataclasses import dataclass +from typing import Optional + + +COACHING_SYSTEM_PROMPT = """You are a personal voice coach and communication advisor named VoiceCoach. +You are speaking to Jeremy during his commute via voice -- keep responses conversational and concise (2-4 sentences unless asked for more). + +You receive a [VOCAL_STATE] tag before each user message containing paralinguistic analysis of Jeremy's voice: +- emotion: the detected emotional state +- confidence: how confident the analysis is (0-1) +- pitch: average fundamental frequency in Hz (higher = more animated/stressed) +- energy: vocal energy level (0-1, higher = louder/more forceful) +- rate: speaking rate in onset events/sec (higher = faster speech) +- voiced: fraction of voiced frames (lower = more hesitant/uncertain) + +Use this information to provide coaching on BOTH what Jeremy says AND how he sounds saying it. +Important: Never quote the raw numbers. Translate them into natural observations. +Example: "I notice some tension in your voice..." not "Your pitch was 215hz..." + +{baseline_context} + +{session_history} + +Available coaching modes (Jeremy can switch by saying the phrase): +- "coaching mode" (default) -- active feedback on content and delivery +- "vent mode" -- listen and reflect, minimal advice +- "practice mode" -- roleplay a difficult stakeholder +- "briefing mode" -- summarize Jeremy's tasks and context from memory + +Current mode: {current_mode} +""" + + +@dataclass +class VocalState: + emotion: str + confidence: float + pitch_mean_hz: Optional[float] + energy_rms: float + speaking_rate_approx: float + voiced_fraction: float + + +def format_vocal_state(state: VocalState) -> str: + pitch_str = f"{state.pitch_mean_hz:.0f}hz" if state.pitch_mean_hz else "unknown" + return ( + f"[VOCAL_STATE " + f"emotion={state.emotion} " + f"confidence={state.confidence:.2f} " + f"pitch={pitch_str} " + f"energy={state.energy_rms:.2f} " + f"rate={state.speaking_rate_approx:.1f}ops " + f"voiced={state.voiced_fraction:.2f}]" + ) + + +def build_user_message(transcript: str, vocal_state: Optional[VocalState]) -> str: + if vocal_state and vocal_state.confidence > 0.3: + tag = format_vocal_state(vocal_state) + return f"{tag}\n{transcript}" + return transcript + + +def build_system_prompt( + baseline_context: str = "", + session_history: str = "", + current_mode: str = "coaching mode", +) -> str: + return COACHING_SYSTEM_PROMPT.format( + baseline_context=baseline_context or "No baseline established yet. First session: ask Jeremy to read a short neutral calibration script.", + session_history=session_history or "No previous session history.", + current_mode=current_mode, + ) diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/requirements.txt b/compose/hf-pdocker-01/voicecoach/pipecat-agent/requirements.txt new file mode 100644 index 0000000..c53c6d7 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/requirements.txt @@ -0,0 +1,7 @@ +pipecat-ai[livekit,silero,openai]==0.0.48 +httpx==0.27.0 +python-dotenv==1.0.1 +structlog==24.4.0 +livekit-api==0.7.1 +fastapi==0.115.0 +uvicorn[standard]==0.30.0 diff --git a/compose/hf-pdocker-01/voicecoach/pipecat-agent/whisper_stt.py b/compose/hf-pdocker-01/voicecoach/pipecat-agent/whisper_stt.py new file mode 100644 index 0000000..31f587e --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/pipecat-agent/whisper_stt.py @@ -0,0 +1,35 @@ +""" +Local STT service using faster-whisper. +Replaces Deepgram with fully self-hosted speech-to-text. +""" +import io +import logging +import wave +import numpy as np +from faster_whisper import WhisperModel + +logger = logging.getLogger(__name__) + + +class LocalWhisperSTT: + """Wraps faster-whisper for use in the Pipecat pipeline.""" + + def __init__(self, model_size: str = "medium", device: str = "cpu", compute_type: str = "int8"): + logger.info(f"Loading faster-whisper model: {model_size} on {device}") + self.model = WhisperModel(model_size, device=device, compute_type=compute_type) + logger.info("faster-whisper model loaded.") + + def transcribe(self, audio_bytes: bytes, sample_rate: int = 16000) -> str: + """Transcribe raw PCM audio bytes to text.""" + audio_array = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0 + + segments, info = self.model.transcribe( + audio_array, + beam_size=5, + language="en", + vad_filter=True, + vad_parameters=dict(min_silence_duration_ms=500), + ) + + text = " ".join(seg.text.strip() for seg in segments) + return text diff --git a/compose/hf-pdocker-01/voicecoach/piper-service/Dockerfile b/compose/hf-pdocker-01/voicecoach/piper-service/Dockerfile new file mode 100644 index 0000000..5661aa1 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/piper-service/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Download Piper voice model at build time from HuggingFace (lessac medium - good quality, fast on CPU) +RUN mkdir -p /app/models && \ + wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx?download=true" \ + -O /app/models/en_US-lessac-medium.onnx && \ + wget -q "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json?download=true" \ + -O /app/models/en_US-lessac-medium.onnx.json + +COPY . . + +EXPOSE 8003 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s \ + CMD python -c "import httpx; httpx.get('http://localhost:8003/health').raise_for_status()" + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8003", "--workers", "1"] diff --git a/compose/hf-pdocker-01/voicecoach/piper-service/main.py b/compose/hf-pdocker-01/voicecoach/piper-service/main.py new file mode 100644 index 0000000..5696ebf --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/piper-service/main.py @@ -0,0 +1,72 @@ +""" +Piper TTS HTTP service. +Self-hosted text-to-speech, internal network only. +""" +import io +import logging +import os +import wave +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response +from pydantic import BaseModel +import piper + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_PATH = os.getenv("PIPER_MODEL", "/app/models/en_US-lessac-medium.onnx") +voice = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global voice + logger.info(f"Loading Piper model: {MODEL_PATH}") + voice = piper.PiperVoice.load(MODEL_PATH) + logger.info("Piper TTS ready.") + yield + logger.info("Shutting down.") + + +app = FastAPI(title="Piper TTS Service", lifespan=lifespan) + + +class TTSRequest(BaseModel): + text: str + speaker_id: int = None + length_scale: float = 1.0 + sentence_silence: float = 0.2 + + +@app.get("/health") +async def health(): + if voice is None: + raise HTTPException(status_code=503, detail="Model not loaded") + return {"status": "ok", "model": os.path.basename(MODEL_PATH)} + + +@app.post("/synthesize") +async def synthesize(req: TTSRequest): + if voice is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + if not req.text.strip(): + raise HTTPException(status_code=400, detail="Empty text") + + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + synth_kwargs = { + "length_scale": req.length_scale, + "sentence_silence": req.sentence_silence, + } + if req.speaker_id is not None: + synth_kwargs["speaker_id"] = req.speaker_id + voice.synthesize(req.text, wf, **synth_kwargs) + + return Response( + content=buf.getvalue(), + media_type="audio/wav", + headers={"Content-Disposition": "inline; filename=speech.wav"}, + ) diff --git a/compose/hf-pdocker-01/voicecoach/piper-service/requirements.txt b/compose/hf-pdocker-01/voicecoach/piper-service/requirements.txt new file mode 100644 index 0000000..fb05cc6 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/piper-service/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.0 +piper-tts==1.2.0 +httpx==0.27.0 +numpy==1.26.4 diff --git a/compose/hf-pdocker-01/voicecoach/speechbrain-service/Dockerfile b/compose/hf-pdocker-01/voicecoach/speechbrain-service/Dockerfile new file mode 100644 index 0000000..bfd7403 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/speechbrain-service/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + libsndfile1 \ + ffmpeg \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download the SpeechBrain model at build time +RUN python -c "\ +from speechbrain.inference.interfaces import foreign_class; \ +foreign_class( \ + source='speechbrain/emotion-recognition-wav2vec2-IEMOCAP', \ + pymodule_file='custom_interface.py', \ + classname='CustomEncoderWav2vec2Classifier', \ + savedir='/app/models', \ + run_opts={'device': 'cpu'}, \ +); \ +print('Model downloaded successfully.')" + +COPY . . + +EXPOSE 8001 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=90s \ + CMD python -c "import httpx; httpx.get('http://localhost:8001/health').raise_for_status()" + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"] diff --git a/compose/hf-pdocker-01/voicecoach/speechbrain-service/analyzer.py b/compose/hf-pdocker-01/voicecoach/speechbrain-service/analyzer.py new file mode 100644 index 0000000..42d166a --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/speechbrain-service/analyzer.py @@ -0,0 +1,119 @@ +""" +SpeechBrain + librosa vocal analysis. +Performs emotion classification and acoustic feature extraction from audio. +""" +import io +import logging +import numpy as np +import soundfile as sf +import librosa +from speechbrain.inference.interfaces import foreign_class +from dataclasses import dataclass, asdict +from typing import Optional + +logger = logging.getLogger(__name__) + +SAMPLE_RATE = 16000 +MIN_DURATION_SECONDS = 0.5 + + +@dataclass +class VocalAnalysis: + emotion: str + emotion_confidence: float + emotion_scores: dict + pitch_mean_hz: Optional[float] + pitch_variance: Optional[float] + energy_rms: float + speaking_rate_approx: float + voiced_fraction: float + duration_seconds: float + analysis_error: Optional[str] = None + + +class VocalAnalyzer: + """Loads models once at startup and provides analysis per audio chunk.""" + + def __init__(self, model_dir: str = "/app/models"): + logger.info("Loading SpeechBrain emotion model...") + self.classifier = foreign_class( + source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", + pymodule_file="custom_interface.py", + classname="CustomEncoderWav2vec2Classifier", + savedir=model_dir, + run_opts={"device": "cpu"}, + ) + logger.info("SpeechBrain model loaded successfully.") + + def analyze(self, audio_bytes: bytes, content_type: str = "audio/wav") -> VocalAnalysis: + try: + audio_buf = io.BytesIO(audio_bytes) + audio, sr = sf.read(audio_buf, dtype="float32") + + if audio.ndim > 1: + audio = audio.mean(axis=1) + + if sr != SAMPLE_RATE: + audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE) + sr = SAMPLE_RATE + + duration = len(audio) / sr + + if duration < MIN_DURATION_SECONDS: + return VocalAnalysis( + emotion="neutral", emotion_confidence=0.0, emotion_scores={}, + pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0, + speaking_rate_approx=0.0, voiced_fraction=0.0, + duration_seconds=duration, + analysis_error=f"Audio too short: {duration:.2f}s < {MIN_DURATION_SECONDS}s minimum", + ) + + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + sf.write(tmp.name, audio, sr) + tmp_path = tmp.name + + try: + out_prob, score, index, text_lab = self.classifier.classify_file(tmp_path) + emotion = text_lab[0] if isinstance(text_lab, list) else text_lab + confidence = float(score[0]) if hasattr(score, "__iter__") else float(score) + labels = ["neu", "hap", "sad", "ang", "fru"] + prob_list = out_prob[0].tolist() if hasattr(out_prob[0], "tolist") else list(out_prob[0]) + emotion_scores = {l: round(p, 4) for l, p in zip(labels, prob_list)} + finally: + os.unlink(tmp_path) + + f0, voiced_flag, _ = librosa.pyin( + audio, fmin=librosa.note_to_hz("C2"), fmax=librosa.note_to_hz("C7"), sr=sr, + ) + + voiced_f0 = f0[voiced_flag] if voiced_flag is not None else np.array([]) + pitch_mean = float(np.mean(voiced_f0)) if len(voiced_f0) > 0 else None + pitch_var = float(np.var(voiced_f0)) if len(voiced_f0) > 1 else None + voiced_fraction = float(np.sum(voiced_flag) / len(voiced_flag)) if voiced_flag is not None else 0.0 + + rms = librosa.feature.rms(y=audio) + energy_rms = float(np.mean(rms)) + + onset_frames = librosa.onset.onset_detect(y=audio, sr=sr) + speaking_rate = len(onset_frames) / duration if duration > 0 else 0.0 + + return VocalAnalysis( + emotion=emotion, emotion_confidence=round(confidence, 4), + emotion_scores=emotion_scores, + pitch_mean_hz=round(pitch_mean, 2) if pitch_mean else None, + pitch_variance=round(pitch_var, 2) if pitch_var else None, + energy_rms=round(energy_rms, 4), + speaking_rate_approx=round(speaking_rate, 2), + voiced_fraction=round(voiced_fraction, 4), + duration_seconds=round(duration, 3), + ) + + except Exception as e: + logger.exception(f"Analysis failed: {e}") + return VocalAnalysis( + emotion="neutral", emotion_confidence=0.0, emotion_scores={}, + pitch_mean_hz=None, pitch_variance=None, energy_rms=0.0, + speaking_rate_approx=0.0, voiced_fraction=0.0, + duration_seconds=0.0, analysis_error=str(e), + ) diff --git a/compose/hf-pdocker-01/voicecoach/speechbrain-service/main.py b/compose/hf-pdocker-01/voicecoach/speechbrain-service/main.py new file mode 100644 index 0000000..ae224f5 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/speechbrain-service/main.py @@ -0,0 +1,66 @@ +""" +FastAPI service wrapping VocalAnalyzer. +Internal-only -- not exposed via Traefik. +""" +import logging +import os +from contextlib import asynccontextmanager +from fastapi import FastAPI, UploadFile, File, HTTPException +from fastapi.responses import JSONResponse +from dataclasses import asdict + +from analyzer import VocalAnalyzer + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +MODEL_DIR = os.getenv("MODEL_DIR", "/app/models") +analyzer: VocalAnalyzer = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global analyzer + logger.info("Starting VocalAnalyzer...") + analyzer = VocalAnalyzer(model_dir=MODEL_DIR) + logger.info("VocalAnalyzer ready.") + yield + logger.info("Shutting down.") + + +app = FastAPI(title="SpeechBrain Vocal Analysis Service", lifespan=lifespan) + + +@app.get("/health") +async def health(): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + return {"status": "ok", "model": "emotion-recognition-wav2vec2-IEMOCAP"} + + +@app.post("/analyze") +async def analyze(audio: UploadFile = File(...)): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + content = await audio.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="Empty audio file") + + result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav") + return JSONResponse(content=asdict(result)) + + +@app.post("/calibrate") +async def calibrate(audio: UploadFile = File(...)): + if analyzer is None: + raise HTTPException(status_code=503, detail="Model not loaded") + + content = await audio.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="Empty audio file") + + result = analyzer.analyze(content, content_type=audio.content_type or "audio/wav") + result_dict = asdict(result) + result_dict["is_baseline"] = True + return JSONResponse(content=result_dict) diff --git a/compose/hf-pdocker-01/voicecoach/speechbrain-service/requirements.txt b/compose/hf-pdocker-01/voicecoach/speechbrain-service/requirements.txt new file mode 100644 index 0000000..b045065 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/speechbrain-service/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.0 +speechbrain==1.0.0 +huggingface_hub==0.19.4 +transformers==4.40.2 +torch==2.3.0 +torchaudio==2.3.0 +librosa==0.10.2 +soundfile==0.12.1 +numpy==1.26.4 +httpx==0.27.0 +python-multipart==0.0.9 diff --git a/compose/hf-pdocker-01/voicecoach/web-client/Dockerfile b/compose/hf-pdocker-01/voicecoach/web-client/Dockerfile new file mode 100644 index 0000000..f7a5433 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/web-client/Dockerfile @@ -0,0 +1,8 @@ +FROM nginx:alpine + +COPY index.html /usr/share/nginx/html/ +COPY app.js /usr/share/nginx/html/ +COPY style.css /usr/share/nginx/html/ +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 diff --git a/compose/hf-pdocker-01/voicecoach/web-client/app.js b/compose/hf-pdocker-01/voicecoach/web-client/app.js new file mode 100644 index 0000000..4c1c121 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/web-client/app.js @@ -0,0 +1,158 @@ +const LIVEKIT_URL = "wss://voice.hobbs.farm"; +let room = null; + +function log(msg, level) { + const el = document.getElementById("logOutput"); + const time = new Date().toLocaleTimeString(); + const cls = level === "error" ? "log-error" : level === "warn" ? "log-warn" : "log-info"; + el.innerHTML += `
${time} ${msg}
`; + el.scrollTop = el.scrollHeight; +} + +function clearLog() { + document.getElementById("logOutput").innerHTML = ""; +} + +async function connect() { + setStatus("connecting", "Connecting..."); + log("Requesting token from server..."); + try { + const response = await fetch("/api/token", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ identity: "jeremy-" + Date.now() }), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error("Token request failed: " + response.status + " " + text); + } + const data = await response.json(); + log("Token received. LiveKit URL: " + data.url); + + room = new LivekitClient.Room({ + audioCaptureDefaults: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + room.on(LivekitClient.RoomEvent.Connected, () => { + setStatus("connected", "Connected"); + document.getElementById("connectBtn").style.display = "none"; + document.getElementById("disconnectBtn").style.display = "block"; + log("Connected to LiveKit room"); + log("Participants in room: " + room.remoteParticipants.size); + if (room.remoteParticipants.size === 0) { + log("No agent participant yet - waiting for voicecoach-agent to join...", "warn"); + } + }); + + room.on(LivekitClient.RoomEvent.Disconnected, (reason) => { + setStatus("disconnected", "Disconnected"); + document.getElementById("connectBtn").style.display = "block"; + document.getElementById("disconnectBtn").style.display = "none"; + log("Disconnected from LiveKit" + (reason ? ": " + reason : ""), "warn"); + }); + + room.on(LivekitClient.RoomEvent.ParticipantConnected, (participant) => { + log("Participant joined: " + participant.identity); + }); + + room.on(LivekitClient.RoomEvent.ParticipantDisconnected, (participant) => { + log("Participant left: " + participant.identity, "warn"); + }); + + room.on(LivekitClient.RoomEvent.TrackSubscribed, (track, publication, participant) => { + log("Track subscribed: " + track.kind + " from " + participant.identity); + if (track.kind === LivekitClient.Track.Kind.Audio) { + const element = track.attach(); + document.body.appendChild(element); + element.play().then(() => { + log("Audio playback started"); + }).catch((err) => { + log("Audio play failed: " + err.message, "error"); + }); + } + }); + + room.on(LivekitClient.RoomEvent.TrackPublished, (publication, participant) => { + log("Track published by " + participant.identity + ": " + publication.kind); + }); + + room.on(LivekitClient.RoomEvent.TrackUnsubscribed, (track, publication, participant) => { + log("Track unsubscribed: " + track.kind + " from " + participant.identity, "warn"); + }); + + room.on(LivekitClient.RoomEvent.DataReceived, (payload, participant) => { + try { + const msg = new TextDecoder().decode(payload); + log("Data from " + (participant ? participant.identity : "server") + ": " + msg); + } catch (e) { + log("Data received (binary, " + payload.byteLength + " bytes)"); + } + }); + + room.on(LivekitClient.RoomEvent.ConnectionQualityChanged, (quality, participant) => { + if (participant.isLocal) { + log("Connection quality: " + quality); + } + }); + + room.on(LivekitClient.RoomEvent.MediaDevicesError, (err) => { + log("Media device error: " + err.message, "error"); + }); + + room.on(LivekitClient.RoomEvent.RoomMetadataChanged, (metadata) => { + log("Room metadata changed: " + metadata); + }); + + log("Connecting to " + LIVEKIT_URL + "..."); + await room.connect(data.url || LIVEKIT_URL, data.token); + + log("Enabling microphone..."); + await room.localParticipant.setMicrophoneEnabled(true); + log("Microphone enabled. Speak now."); + + // Log local tracks + room.localParticipant.audioTrackPublications.forEach((pub) => { + log("Local audio track: " + pub.trackSid + " muted=" + pub.isMuted); + }); + + } catch (err) { + setStatus("disconnected", "Error: " + err.message); + log("Connection error: " + err.message, "error"); + console.error(err); + } +} + +async function disconnect() { + if (room) { + log("Disconnecting..."); + await room.disconnect(); + room = null; + } +} + +function setMode(mode) { + document.getElementById("modeDisplay").textContent = "Mode: " + mode; + log("Mode changed to: " + mode); + if (room && room.localParticipant) { + const encoder = new TextEncoder(); + room.localParticipant.publishData( + encoder.encode(JSON.stringify({ type: "mode_change", mode })), + { reliable: true } + ); + } +} + +function setStatus(state, text) { + const el = document.getElementById("status"); + el.className = "status " + state; + el.textContent = text; +} + +document.getElementById("connectBtn").addEventListener("click", connect); +document.getElementById("disconnectBtn").addEventListener("click", disconnect); + +log("VoiceCoach client loaded. Ready to connect."); diff --git a/compose/hf-pdocker-01/voicecoach/web-client/index.html b/compose/hf-pdocker-01/voicecoach/web-client/index.html new file mode 100644 index 0000000..5af0648 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/web-client/index.html @@ -0,0 +1,46 @@ + + + + + + VoiceCoach + + + +
+
+

VoiceCoach

+
Disconnected
+
+ +
+ + +
+ +
Mode: coaching
+ +
+

Press "Start Session" to connect your voice.

+
+ +
+ + + + +
+ +
+
+ Log + +
+
+
+
+ + + + + diff --git a/compose/hf-pdocker-01/voicecoach/web-client/nginx.conf b/compose/hf-pdocker-01/voicecoach/web-client/nginx.conf new file mode 100644 index 0000000..e15bc6d --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/web-client/nginx.conf @@ -0,0 +1,15 @@ +server { + listen 80; + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://pipecat-agent:8080/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/compose/hf-pdocker-01/voicecoach/web-client/style.css b/compose/hf-pdocker-01/voicecoach/web-client/style.css new file mode 100644 index 0000000..52ca288 --- /dev/null +++ b/compose/hf-pdocker-01/voicecoach/web-client/style.css @@ -0,0 +1,131 @@ +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: #1a1a2e; + color: #eee; + min-height: 100vh; + display: flex; + justify-content: center; + align-items: center; +} + +.container { + max-width: 420px; + width: 100%; + padding: 24px; +} + +.header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 24px; +} + +h1 { font-size: 24px; font-weight: 600; } + +.status { + padding: 4px 12px; + border-radius: 12px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; +} +.status.disconnected { background: #444; color: #aaa; } +.status.connecting { background: #553; color: #ffa; } +.status.connected { background: #253; color: #5f5; } + +.controls { margin-bottom: 20px; } + +.btn { + width: 100%; + padding: 14px; + border: none; + border-radius: 8px; + font-size: 16px; + font-weight: 600; + cursor: pointer; + transition: opacity 0.2s; +} +.btn:hover { opacity: 0.85; } +.btn-primary { background: #4a6cf7; color: #fff; } +.btn-danger { background: #f74a4a; color: #fff; } + +.mode-display { + text-align: center; + font-size: 13px; + color: #888; + margin-bottom: 16px; +} + +.transcript { + background: #16213e; + border-radius: 8px; + padding: 16px; + min-height: 200px; + max-height: 400px; + overflow-y: auto; + margin-bottom: 16px; +} +.transcript .hint { color: #666; font-style: italic; } + +.mode-buttons { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} +.mode-btn { + padding: 10px; + background: #16213e; + border: 1px solid #333; + border-radius: 6px; + color: #ccc; + font-size: 13px; + cursor: pointer; + transition: background 0.2s; +} +.mode-btn:hover { background: #1a2744; border-color: #4a6cf7; } + +.log-panel { + margin-top: 16px; + background: #0d1117; + border: 1px solid #333; + border-radius: 8px; + overflow: hidden; +} +.log-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 6px 12px; + background: #161b22; + font-size: 12px; + font-weight: 600; + color: #888; + text-transform: uppercase; + letter-spacing: 0.5px; +} +.log-clear { + background: none; + border: 1px solid #444; + border-radius: 4px; + color: #888; + font-size: 11px; + padding: 2px 8px; + cursor: pointer; +} +.log-clear:hover { border-color: #666; color: #ccc; } +.log-output { + padding: 8px 12px; + max-height: 200px; + overflow-y: auto; + font-family: 'SF Mono', 'Fira Code', monospace; + font-size: 11px; + line-height: 1.6; +} +.log-output div { word-break: break-all; } +.log-time { color: #555; margin-right: 6px; } +.log-info { color: #8b949e; } +.log-warn { color: #d29922; } +.log-error { color: #f85149; } diff --git a/stacks/traefik/dynamic/friday-voice.yml b/stacks/traefik/dynamic/friday-voice.yml new file mode 100644 index 0000000..5971910 --- /dev/null +++ b/stacks/traefik/dynamic/friday-voice.yml @@ -0,0 +1,31 @@ +http: + routers: + friday-voice: + rule: "Host(`coach.hobbs.farm`)" + entryPoints: + - websecure + middlewares: + - security-headers-allow-mic + service: friday-voice + tls: + certResolver: letsencrypt + friday-voice-http: + rule: "Host(`coach.hobbs.farm`)" + entryPoints: + - web + middlewares: + - friday-voice-https-redirect + service: friday-voice + + middlewares: + friday-voice-https-redirect: + redirectScheme: + scheme: https + permanent: true + + services: + friday-voice: + loadBalancer: + servers: + # friday-voice runs on the .150 Windows box (Docker Desktop), port 8320 + - url: "http://192.168.86.150:8320" diff --git a/stacks/traefik/dynamic/livekit.yml b/stacks/traefik/dynamic/livekit.yml new file mode 100644 index 0000000..da85880 --- /dev/null +++ b/stacks/traefik/dynamic/livekit.yml @@ -0,0 +1,29 @@ +http: + routers: + livekit: + rule: "Host(`voice.hobbs.farm`)" + entryPoints: + - websecure + service: livekit + tls: + certResolver: letsencrypt + livekit-http: + rule: "Host(`voice.hobbs.farm`)" + entryPoints: + - web + middlewares: + - livekit-https-redirect + service: livekit + + middlewares: + livekit-https-redirect: + redirectScheme: + scheme: https + permanent: true + + services: + livekit: + loadBalancer: + servers: + - url: "http://192.168.86.150:7880" + passHostHeader: true diff --git a/stacks/traefik/dynamic/voicecoach.yml b/stacks/traefik/dynamic/voicecoach.yml new file mode 100644 index 0000000..a3d594c --- /dev/null +++ b/stacks/traefik/dynamic/voicecoach.yml @@ -0,0 +1,30 @@ +http: + routers: + voicecoach: + rule: "Host(`coach.hobbs.farm`)" + entryPoints: + - websecure + middlewares: + - security-headers-allow-mic + service: voicecoach + tls: + certResolver: letsencrypt + voicecoach-http: + rule: "Host(`coach.hobbs.farm`)" + entryPoints: + - web + middlewares: + - voicecoach-https-redirect + service: voicecoach + + middlewares: + voicecoach-https-redirect: + redirectScheme: + scheme: https + permanent: true + + services: + voicecoach: + loadBalancer: + servers: + - url: "http://192.168.86.150:8085"