diff --git a/llm/README.md b/llm/README.md new file mode 100644 index 0000000..5228638 --- /dev/null +++ b/llm/README.md @@ -0,0 +1,15 @@ +# LLM + +Text generation with [Electron](https://docs.smallest.ai/waves/api-reference), Smallest AI's LLM, via an OpenAI-compatible `chat/completions` endpoint. + +## Endpoint + +``` +https://api.smallest.ai/waves/v1/chat/completions +``` + +Use the stock `openai` package: `OpenAI(api_key=..., base_url="https://api.smallest.ai/waves/v1")` with `model="electron"`. Standard chat completions, streaming, and function calling all work unchanged. + +## Examples + +- [Tool Calling](./tool-calling/): function calling with the standard OpenAI tools schema. The model requests a tool, you execute locally, the model composes the answer. diff --git a/llm/tool-calling/.env.sample b/llm/tool-calling/.env.sample new file mode 100644 index 0000000..6277475 --- /dev/null +++ b/llm/tool-calling/.env.sample @@ -0,0 +1,3 @@ +# Smallest AI API Key +# Get yours at https://smallest.ai/console +SMALLEST_API_KEY=your-smallest-api-key-here diff --git a/llm/tool-calling/README.md b/llm/tool-calling/README.md new file mode 100644 index 0000000..5d0bdf4 --- /dev/null +++ b/llm/tool-calling/README.md @@ -0,0 +1,50 @@ +# Tool Calling (Electron) + +Function calling with the Electron LLM through the OpenAI-compatible endpoint. Define tools with the standard OpenAI schema, let the model request a call, execute it locally, and feed the result back for the final answer. + +## Try It + +```bash +uv run tool_calling.py "What's the weather in Mumbai?" +``` + +Output: + +``` +Model called get_weather({'city': 'Mumbai'}) + +It's currently partly cloudy in Mumbai at about 29°C. +``` + +## Requirements + +> Base dependencies are installed via the root `requirements.txt` (includes `openai`). See the [main README](../../README.md#usage) for setup. Add `SMALLEST_API_KEY` to your `.env` (see `.env.sample`). + +## How It Works + +The endpoint is OpenAI-compatible, so the stock `openai` package works as-is: + +```python +from openai import OpenAI + +client = OpenAI( + api_key=os.environ["SMALLEST_API_KEY"], + base_url="https://api.smallest.ai/waves/v1", +) +``` + +The two-step tool loop: + +1. `chat.completions.create(model="electron", messages=..., tools=...)`: the model either answers directly or returns `tool_calls`. +2. If it requested tools: execute each one locally (here, a stubbed `get_weather(city)`), append the assistant message and a `role: "tool"` message with the JSON result, then call `create` again. The model composes the final answer from the tool output. + +Non-streaming for clarity. The same endpoint also speaks the plain `chat/completions` protocol, so everything else the `openai` client supports (streaming, system prompts, temperature) works the same way. + +## API Reference + +- [Waves API Reference](https://docs.smallest.ai/waves/api-reference) + +## Next Steps + +- [Agent with Tools](../../voice-agents/agent_with_tools/): tool calling inside a full voice agent +- [BYOM](../../voice-agents/byom/): point voice agents at any OpenAI-compatible LLM diff --git a/llm/tool-calling/tool_calling.py b/llm/tool-calling/tool_calling.py new file mode 100644 index 0000000..f48a789 --- /dev/null +++ b/llm/tool-calling/tool_calling.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +""" +Smallest AI Electron LLM - Tool Calling + +Function calling with the Electron LLM via the OpenAI-compatible +chat/completions endpoint. The model decides when to call get_weather(city), +the script runs it locally, and the model composes the final answer from +the tool result. + +Usage: python tool_calling.py "What's the weather in Mumbai?" +""" + +import json +import os +import sys + +from dotenv import load_dotenv +from openai import OpenAI + +load_dotenv() + +MODEL = "electron" +BASE_URL = "https://api.smallest.ai/waves/v1" + +DEFAULT_QUESTION = "What's the weather in Mumbai right now?" + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name, e.g. Mumbai", + } + }, + "required": ["city"], + }, + }, + } +] + + +def get_weather(city: str) -> dict: + """Stub implementation. Swap in a real weather API call.""" + return {"city": city, "condition": "partly cloudy", "temperature_c": 29} + + +def main(): + api_key = os.environ.get("SMALLEST_API_KEY") + if not api_key: + print("Error: SMALLEST_API_KEY environment variable not set", file=sys.stderr) + sys.exit(1) + + question = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_QUESTION + client = OpenAI(api_key=api_key, base_url=BASE_URL) + + messages = [{"role": "user", "content": question}] + + # Step 1: let the model decide whether it needs the tool. + response = client.chat.completions.create( + model=MODEL, + messages=messages, + tools=TOOLS, + ) + message = response.choices[0].message + + if message.tool_calls: + messages.append(message) + + # Step 2: execute each requested tool locally and send back the results. + for call in message.tool_calls: + args = json.loads(call.function.arguments) + print(f"Model called {call.function.name}({args})") + + result = get_weather(**args) + messages.append({ + "role": "tool", + "tool_call_id": call.id, + "content": json.dumps(result), + }) + + response = client.chat.completions.create( + model=MODEL, + messages=messages, + tools=TOOLS, + ) + message = response.choices[0].message + + print(f"\n{message.content}") + + +if __name__ == "__main__": + main() diff --git a/speech-to-speech/README.md b/speech-to-speech/README.md index 3911030..d590d7d 100644 --- a/speech-to-speech/README.md +++ b/speech-to-speech/README.md @@ -22,4 +22,5 @@ with client.waves.speech_to_speech.connect() as socket: ## Examples +- [Python Quickstart](./python-quickstart/): minimal headless client. Streams a WAV file as paced live audio, prints the reply transcript, saves the spoken response to `reply.wav`. - [Hydra Realtime Demo](./hydra-realtime-demo/) — Next.js browser client with multi-agent presets (companion, restaurant, banking), client-side tool calling, a wire-log of every WebSocket frame, and persona/voice editing. Mirror of [smallest-inc/hydra_agents](https://github.com/smallest-inc/hydra_agents). diff --git a/speech-to-speech/hydra-realtime-demo/README.md b/speech-to-speech/hydra-realtime-demo/README.md index 4fa1317..beecf76 100644 --- a/speech-to-speech/hydra-realtime-demo/README.md +++ b/speech-to-speech/hydra-realtime-demo/README.md @@ -14,7 +14,7 @@ Talk to Hydra in your browser, watch every WebSocket frame as it lands, and swit - **Wire-log tab** — every `session.*`, `input_audio_buffer.*`, `response.*`, `conversation.item.*`, and `error` event, in & out, with a JSON inspector. - **Multi-agent presets** — a friendly companion, a burger-restaurant phone agent (`Smallest Kitchen`), and a banking concierge (`NovaBank`). All tools execute locally — Hydra is a pure voice engine. - **Beautiful, audio-reactive orb** that pulses with the live RMS level of whichever side is currently active. -- **Persona / voice editing** — change the system prompt, pick from six Waves voices, toggle whether the agent speaks first. +- **Persona / voice editing** — change the system prompt, pick from six Hydra voices, toggle whether the agent speaks first. - **Zero server** — connects browser → Hydra directly. Your API key never leaves your browser. --- @@ -61,12 +61,12 @@ ws open ← { "type": "session.created", "session_id": "…" } → { "type": "session.configure", "session": { "instructions": "you are a friendly voice assistant.", - "voice": "wren", + "voice": "aria", "tools": [], "generate_initial_response": false }} ← { "type": "session.configured", "session": { - "voice": "wren", + "voice": "aria", "input_audio_format": "pcm16", "input_audio_sample_rate": 16000, "output_audio_format": "pcm16", diff --git a/speech-to-speech/hydra-realtime-demo/src/app/agents/presets.ts b/speech-to-speech/hydra-realtime-demo/src/app/agents/presets.ts index 45dbbbf..791d53a 100644 --- a/speech-to-speech/hydra-realtime-demo/src/app/agents/presets.ts +++ b/speech-to-speech/hydra-realtime-demo/src/app/agents/presets.ts @@ -34,7 +34,7 @@ export const AGENT_PRESETS: AgentPreset[] = [ tagline: "A friendly, curious voice assistant.", emoji: "✦", category: "Featured", - voice: "wren", + voice: "aria", instructions: `You are Hydra, a friendly real-time voice companion built on Smallest.ai. ` + `Be warm, concise, and conversational. Speak naturally — you are a voice, ` + @@ -48,7 +48,7 @@ export const AGENT_PRESETS: AgentPreset[] = [ tagline: "A burger-joint phone agent that takes orders.", emoji: "🍔", category: "Demos", - voice: "sloane", + voice: "maya", instructions: `You are the phone agent for Smallest Kitchen, a popular burger restaurant. ` + `Take the customer's order conversationally — never recite the whole menu unless they ask. ` + @@ -64,7 +64,7 @@ export const AGENT_PRESETS: AgentPreset[] = [ tagline: "A banking concierge that handles balances, transfers, and cards.", emoji: "🏦", category: "Demos", - voice: "reed", + voice: "sterling", instructions: `You are a NovaBank phone concierge. Help the customer with balances, ` + `recent transactions, transfers between checking and savings, blocking ` + diff --git a/speech-to-speech/hydra-realtime-demo/src/app/components/ControlPanel.tsx b/speech-to-speech/hydra-realtime-demo/src/app/components/ControlPanel.tsx index 86a4bce..283d242 100644 --- a/speech-to-speech/hydra-realtime-demo/src/app/components/ControlPanel.tsx +++ b/speech-to-speech/hydra-realtime-demo/src/app/components/ControlPanel.tsx @@ -3,12 +3,12 @@ import { AGENT_PRESETS, findPreset } from "@/app/agents/presets"; const VOICES = [ - { value: "wren", label: "Wren" }, - { value: "sloane", label: "Sloane" }, - { value: "marlowe", label: "Marlowe" }, - { value: "reed", label: "Reed" }, - { value: "knox", label: "Knox" }, - { value: "tate", label: "Tate" }, + { value: "aria", label: "Aria" }, + { value: "maya", label: "Maya" }, + { value: "marin", label: "Marin" }, + { value: "sterling", label: "Sterling" }, + { value: "kai", label: "Kai" }, + { value: "zoe", label: "Zoe" }, ]; export interface ControlPanelProps { diff --git a/speech-to-speech/python-quickstart/.env.sample b/speech-to-speech/python-quickstart/.env.sample new file mode 100644 index 0000000..6277475 --- /dev/null +++ b/speech-to-speech/python-quickstart/.env.sample @@ -0,0 +1,3 @@ +# Smallest AI API Key +# Get yours at https://smallest.ai/console +SMALLEST_API_KEY=your-smallest-api-key-here diff --git a/speech-to-speech/python-quickstart/README.md b/speech-to-speech/python-quickstart/README.md new file mode 100644 index 0000000..d3a7425 --- /dev/null +++ b/speech-to-speech/python-quickstart/README.md @@ -0,0 +1,73 @@ +# Python Quickstart (Hydra) + +Minimal headless Hydra client. Streams a WAV file to the realtime speech-to-speech endpoint as if it were live microphone audio, prints the assistant's transcript as it streams, and saves the spoken reply to `reply.wav`. + +## Try It + +```bash +uv run quickstart.py path/to/audio.wav +``` + +Output: + +``` +Streaming 3.2s of audio... +[speech started] +[speech stopped] +Sure, I can help with that. What would you like to know? +Saved reply to reply.wav +``` + +## Requirements + +> Base dependencies are installed via the root `requirements.txt`. See the [main README](../../README.md#usage) for setup. Add `SMALLEST_API_KEY` to your `.env` (see `.env.sample`). + +Input must be a 16-bit PCM WAV. Any sample rate and channel count works; the script downmixes to mono and naively resamples to 16 kHz. + +## How It Works + +1. Connect to: + + ``` + wss://api.smallest.ai/waves/v1/s2s?model=hydra&api_key= + ``` + +2. Configure the session: + + ```json + { + "type": "session.configure", + "session": { + "instructions": "You are a helpful voice assistant...", + "voice": "wren", + "input_audio_sample_rate": 16000, + "output_audio_sample_rate": 16000 + } + } + ``` + +3. Stream the file's PCM16 as `{"type": "input_audio_buffer.append", "audio": ""}` chunks of 100 ms, paced at real time, followed by about 1.2 s of silence chunks so the server-side VAD detects end of speech and closes the turn. + +4. Read events concurrently while sending: + + | Event | Meaning | + |-------|---------| + | `input_audio_buffer.speech_started` / `speech_stopped` | Server VAD detected your speech | + | `response.output_audio.delta` | Base64 PCM16 reply audio (collected) | + | `response.output_audio_transcript.delta` | Reply transcript text (printed live) | + | `response.done` | Turn finished, stop reading | + +5. The collected reply audio is saved to `reply.wav` (16 kHz, mono, 16-bit). + +## Pacing Matters + +Hydra expects a live audio stream. The script sends one 100 ms chunk every 100 ms of wall-clock time. Dumping the entire file at once gets the socket closed by the server with a "session ended" message, so keep the pacing if you adapt this script. + +## API Reference + +- [Hydra overview and event reference](https://docs.smallest.ai/waves/documentation/speech-to-speech-hydra/overview) +- [Waves API Reference](https://docs.smallest.ai/waves/api-reference) + +## Next Steps + +- [Hydra Realtime Demo](../hydra-realtime-demo/): full browser client with live microphone, agent presets, and tool calling diff --git a/speech-to-speech/python-quickstart/quickstart.py b/speech-to-speech/python-quickstart/quickstart.py new file mode 100644 index 0000000..702a78a --- /dev/null +++ b/speech-to-speech/python-quickstart/quickstart.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +Smallest AI Speech-to-Speech (Hydra) - Python Quickstart + +Minimal headless Hydra client. Streams a WAV file to the realtime endpoint +as if it were live microphone audio, prints the assistant's transcript as it +arrives, and saves the spoken reply to reply.wav. + +Audio is paced at real time (100 ms chunks). Hydra expects a live stream; +dumping the whole file at once gets the session closed by the server. + +Usage: python quickstart.py path/to/audio.wav +""" + +import array +import asyncio +import base64 +import json +import os +import sys +import wave + +import websockets +from dotenv import load_dotenv + +load_dotenv() + +WS_URL = "wss://api.smallest.ai/waves/v1/s2s?model=hydra&api_key={api_key}" + +VOICE = "aria" +INSTRUCTIONS = "You are a helpful voice assistant. Keep replies short and conversational." + +SAMPLE_RATE = 16000 # Hydra input and output, PCM16 mono +CHUNK_MS = 100 +CHUNK_BYTES = SAMPLE_RATE * 2 * CHUNK_MS // 1000 # 3200 bytes per 100 ms +SILENCE_CHUNKS = 12 # ~1.2 s of trailing silence so server VAD closes the turn + +OUTPUT_FILE = "reply.wav" + + +def load_pcm16_mono_16k(path: str) -> bytes: + """Read a WAV file and return PCM16 mono at 16 kHz (naive resample).""" + with wave.open(path, "rb") as f: + channels = f.getnchannels() + width = f.getsampwidth() + rate = f.getframerate() + frames = f.readframes(f.getnframes()) + + if width != 2: + raise SystemExit(f"Error: expected 16-bit PCM WAV, got {width * 8}-bit") + + samples = array.array("h") + samples.frombytes(frames) + + if channels > 1: + samples = samples[0::channels] # keep the first channel + + if rate != SAMPLE_RATE: + # Naive nearest-sample resample. Fine for a demo; use a proper + # resampler (e.g. soxr, librosa) for production audio. + n_out = int(len(samples) * SAMPLE_RATE / rate) + samples = array.array( + "h", (samples[int(i * rate / SAMPLE_RATE)] for i in range(n_out)) + ) + + return samples.tobytes() + + +async def send_audio(ws, pcm: bytes) -> None: + """Stream PCM in 100 ms chunks at real time, then trailing silence.""" + for i in range(0, len(pcm), CHUNK_BYTES): + await ws.send(json.dumps({ + "type": "input_audio_buffer.append", + "audio": base64.b64encode(pcm[i:i + CHUNK_BYTES]).decode(), + })) + await asyncio.sleep(CHUNK_MS / 1000) + + silence = base64.b64encode(b"\x00" * CHUNK_BYTES).decode() + for _ in range(SILENCE_CHUNKS): + await ws.send(json.dumps({ + "type": "input_audio_buffer.append", + "audio": silence, + })) + await asyncio.sleep(CHUNK_MS / 1000) + + +async def run(audio_path: str, api_key: str) -> bytes: + reply_chunks = [] + + async with websockets.connect(WS_URL.format(api_key=api_key)) as ws: + await ws.send(json.dumps({ + "type": "session.configure", + "session": { + "instructions": INSTRUCTIONS, + "voice": VOICE, + "input_audio_sample_rate": SAMPLE_RATE, + "output_audio_sample_rate": SAMPLE_RATE, + }, + })) + + pcm = load_pcm16_mono_16k(audio_path) + print(f"Streaming {len(pcm) / (SAMPLE_RATE * 2):.1f}s of audio...") + sender = asyncio.create_task(send_audio(ws, pcm)) + + try: + async for message in ws: + event = json.loads(message) + event_type = event.get("type") + + if event_type == "input_audio_buffer.speech_started": + print("[speech started]") + + elif event_type == "input_audio_buffer.speech_stopped": + print("[speech stopped]") + + elif event_type == "response.output_audio.delta": + audio_b64 = event.get("delta") or event.get("audio") or "" + reply_chunks.append(base64.b64decode(audio_b64)) + + elif event_type == "response.output_audio_transcript.delta": + print(event.get("delta", ""), end="", flush=True) + + elif event_type == "response.done": + print() + break + + elif event_type == "error": + print(f"\nError: {event}", file=sys.stderr) + sys.exit(1) + finally: + sender.cancel() + + return b"".join(reply_chunks) + + +def save_wav(pcm_data: bytes, path: str) -> None: + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) # 16-bit + f.setframerate(SAMPLE_RATE) + f.writeframes(pcm_data) + + +def main(): + if len(sys.argv) < 2: + print("Usage: python quickstart.py path/to/audio.wav") + sys.exit(1) + + api_key = os.environ.get("SMALLEST_API_KEY") + if not api_key: + print("Error: SMALLEST_API_KEY environment variable not set", file=sys.stderr) + sys.exit(1) + + reply_pcm = asyncio.run(run(sys.argv[1], api_key)) + + if reply_pcm: + save_wav(reply_pcm, OUTPUT_FILE) + print(f"Saved reply to {OUTPUT_FILE}") + else: + print("No reply audio received") + + +if __name__ == "__main__": + main() diff --git a/text-to-speech/sdk-usage/README.md b/text-to-speech/sdk-usage/README.md index 0546415..d82a9c0 100644 --- a/text-to-speech/sdk-usage/README.md +++ b/text-to-speech/sdk-usage/README.md @@ -1,23 +1,72 @@ # SDK Usage -> **Coming Soon** — The Python SDK does not yet support Lightning v3.1. Once it does, this example will cover sync, async, and streaming patterns via the SDK. +Text-to-speech through the `smallestai` Python SDK. One client covers sync, async, and streaming. -## In the Meantime +```bash +pip install smallestai +export SMALLEST_API_KEY="your-key" +``` -All cookbook examples use the Lightning v3.1 REST and WebSocket APIs directly, which work great. See: +## Sync synthesis -- [Getting Started](../getting-started/) — REST API synthesis (Python + JS) -- [Streaming](../streaming/) — SSE and WebSocket streaming (Python + JS) -- [Voices](../voices/) — List and preview all v3.1 voices +```python +from smallestai import SmallestAI -## What This Will Cover +client = SmallestAI() # reads SMALLEST_API_KEY from the environment -- **Sync synthesis** — Simple blocking call, best for scripts and CLIs -- **Async synthesis** — Non-blocking, ideal for web servers and async pipelines -- **Streaming** — WebSocket-based streaming for real-time audio delivery -- **Utility methods** — List voices, get languages, manage cloned voices +audio = b"".join(client.waves.synthesize_tts( + text="Hello from the Python SDK!", + voice_id="sophia", + model="lightning_v3.1", + sample_rate=24000, + output_format="wav", +)) +with open("hello.wav", "wb") as f: + f.write(audio) +``` + +## Async synthesis + +```python +import asyncio +from smallestai import AsyncSmallestAI + +async def main(): + client = AsyncSmallestAI() + chunks = [] + async for chunk in client.waves.synthesize_tts( + text="Async works the same way.", + voice_id="sophia", + model="lightning_v3.1", + ): + chunks.append(chunk) + with open("hello_async.wav", "wb") as f: + f.write(b"".join(chunks)) + +asyncio.run(main()) +``` + +## Streaming over WebSocket + +```python +from smallestai.waves import WavesStreamingTTS, TTSConfig + +config = TTSConfig(voice_id="sophia", model="lightning_v3.1", api_key="YOUR_API_KEY") +streaming_tts = WavesStreamingTTS(config) +for chunk in streaming_tts.synthesize("Streaming audio, chunk by chunk."): + ... # play or buffer each PCM chunk as it arrives +``` + +For SSE streaming use `client.waves.synthesize_sse_tts(...)` with the same payload. + +## Utility methods + +```python +voices = client.waves.get_voices(model="lightning-v3.1") +clones = client.waves.list_voice_clones() +``` ## API Reference - [Python SDK on GitHub](https://github.com/smallest-inc/smallest-python-sdk) -- [Lightning v3.1 API](https://waves-docs.smallest.ai/v4.0.0/content/api-references/lightning-v3.1) +- [TTS documentation](https://docs.smallest.ai/waves/documentation/text-to-speech-lightning/quickstart) diff --git a/text-to-speech/voice-cloning/README.md b/text-to-speech/voice-cloning/README.md index ac8240a..2274451 100644 --- a/text-to-speech/voice-cloning/README.md +++ b/text-to-speech/voice-cloning/README.md @@ -1,28 +1,61 @@ # Voice Cloning -> **Coming Soon** — Instant voice cloning examples are under development and will be added shortly. +Clone any voice from a short audio sample (5-15 seconds) and use it for text-to-speech synthesis. -Clone any voice from a short audio sample (5–15 seconds) and use it for text-to-speech synthesis. +## Instant Clone -## Planned Examples +Upload a short clip, get a usable voice ID back: -- **Instant Clone** — Upload a short audio clip, get a usable voice ID back -- **Clone and Speak** — End-to-end: clone a voice then synthesize speech with it -- **Manage Cloned Voices** — List, preview, and delete your cloned voices +```python +from smallestai import SmallestAI -## API Reference +client = SmallestAI() # reads SMALLEST_API_KEY from the environment + +with open("sample.wav", "rb") as f: + res = client.waves.create_voice_clone( + display_name="My Custom Voice", + file=("sample.wav", f, "audio/wav"), + ) +voice_id = res.data.voice_id +print(f"Voice ID: {voice_id}") +``` -- [Voice Cloning API](https://waves-docs.smallest.ai/v4.0.0/content/api-references/voice-cloning-api) -- [Python SDK — add_voice()](https://github.com/smallest-inc/smallest-python-sdk) +Pass the file as a `(filename, file_object, content_type)` tuple so the upload carries the right file type. -## In the Meantime +## Clone and Speak + +Use the new voice ID like any catalog voice on the unified TTS endpoint: + +```python +audio = b"".join(client.waves.synthesize_tts( + text="Hello from my cloned voice!", + voice_id=voice_id, + model="lightning_v3.1", + sample_rate=24000, +)) +``` -You can clone voices today via the [Smallest AI platform](https://app.smallest.ai) or the Python SDK: +## List Cloned Voices ```python -from smallestai.waves import WavesClient +for clone in client.waves.list_voice_clones().data: + print(clone.voice_id, clone.display_name, clone.status) +``` + +## REST equivalents + +```bash +# Create +curl -X POST "https://api.smallest.ai/waves/v1/voice-cloning" \ + -H "Authorization: Bearer $SMALLEST_API_KEY" \ + -F "displayName=My Custom Voice" \ + -F "file=@sample.wav" -client = WavesClient(api_key="YOUR_API_KEY") -voice = client.add_voice("My Custom Voice", "sample.wav") -print(f"Voice ID: {voice}") +# List +curl "https://api.smallest.ai/waves/v1/voice-cloning" \ + -H "Authorization: Bearer $SMALLEST_API_KEY" ``` + +## API Reference + +- [Voice Cloning guide](https://docs.smallest.ai/waves/documentation/voice-cloning/how-to-vc) diff --git a/text-to-speech/word-timestamps-live/.env.sample b/text-to-speech/word-timestamps-live/.env.sample new file mode 100644 index 0000000..6277475 --- /dev/null +++ b/text-to-speech/word-timestamps-live/.env.sample @@ -0,0 +1,3 @@ +# Smallest AI API Key +# Get yours at https://smallest.ai/console +SMALLEST_API_KEY=your-smallest-api-key-here diff --git a/text-to-speech/word-timestamps-live/README.md b/text-to-speech/word-timestamps-live/README.md new file mode 100644 index 0000000..25fcb53 --- /dev/null +++ b/text-to-speech/word-timestamps-live/README.md @@ -0,0 +1,73 @@ +# Live Word Timestamps + +WebSocket TTS with per-word timing. While the audio streams in, the server also sends a timestamp frame for every word, so you can build live captions, karaoke-style highlighting, or word-level alignment without a separate forced aligner. + +## Try It + +```bash +uv run python/word_timestamps.py "Word timestamps make live captions easy." +``` + +Output: + +``` +[ 0.00s - 0.32s] Word +[ 0.32s - 0.78s] timestamps +[ 0.78s - 1.05s] make +[ 1.05s - 1.38s] live +[ 1.38s - 1.82s] captions +[ 1.82s - 2.20s] easy. + +6 words, 4 audio chunks +Saved to out.wav +``` + +## Requirements + +> Base dependencies are installed via the root `requirements.txt`. See the [main README](../../README.md#usage) for setup. Add `SMALLEST_API_KEY` to your `.env` (see `.env.sample`). + +## How It Works + +1. Connect to `wss://api.smallest.ai/waves/v1/tts/live` with an `Authorization: Bearer ` header. +2. Send one JSON payload with `"word_timestamps": true`, then `{"flush": true}`: + + ```json + { + "text": "Word timestamps make live captions easy.", + "voice_id": "sophia", + "model": "lightning_v3.1", + "sample_rate": 24000, + "word_timestamps": true + } + ``` + +3. The server interleaves three frame types, discriminated by `status`: + + | `status` | Payload | + |----------|---------| + | `chunk` | `data.audio`, base64 PCM16 audio | + | `word_timestamp` | Word timing objects `{id, word, start, end}` (times in seconds) | + | `complete` | Synthesis finished | + + An `error` frame carries the failure detail; the script prints it and exits nonzero. + +4. The script prints a caption line per word as timestamps arrive, joins the PCM chunks, and wraps them with Python's `wave` module into `out.wav` (24 kHz, mono, 16-bit). + +## Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `MODEL` | TTS model | `lightning_v3.1` | +| `VOICE_ID` | Voice to use | `sophia` | +| `SAMPLE_RATE` | Audio sample rate in Hz | `24000` | +| `word_timestamps` | Enable per-word timing frames | `true` | + +## API Reference + +- [Live TTS WebSocket](https://docs.smallest.ai/waves/api-reference/api-reference/text-to-speech/live-tts-web-socket) +- [Waves API Reference](https://docs.smallest.ai/waves/api-reference) + +## Next Steps + +- [Streaming](../streaming/): SSE and WebSocket streaming without timestamps +- [Word-Level Outputs (STT)](../../speech-to-text/word-level-outputs/): word timestamps for transcription instead of synthesis diff --git a/text-to-speech/word-timestamps-live/python/word_timestamps.py b/text-to-speech/word-timestamps-live/python/word_timestamps.py new file mode 100644 index 0000000..8412d76 --- /dev/null +++ b/text-to-speech/word-timestamps-live/python/word_timestamps.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Smallest AI Text-to-Speech - Live Word Timestamps + +Stream TTS audio over WebSocket with per-word timing. Word timestamp frames +arrive alongside the audio chunks, so you can render live captions or +karaoke-style highlighting while the speech is still being generated. + +Usage: python word_timestamps.py "Text to speak" + +Output: +- One caption line per word with start/end times printed as they arrive +- out.wav (24 kHz mono 16-bit) +""" + +import asyncio +import base64 +import json +import os +import sys +import wave + +import websockets +from dotenv import load_dotenv + +load_dotenv() + +MODEL = "lightning_v3.1" +VOICE_ID = "sophia" +SAMPLE_RATE = 24000 +WS_URL = "wss://api.smallest.ai/waves/v1/tts/live" +OUTPUT_FILE = "out.wav" + +DEFAULT_TEXT = "Word timestamps let you caption speech while it is still being generated." + + +def print_word(entry: dict) -> None: + """Print one caption line for a word timing object {id, word, start, end}.""" + start = float(entry["start"]) + end = float(entry["end"]) + print(f"[{start:6.2f}s - {end:6.2f}s] {entry['word']}") + + +async def synthesize(text: str, api_key: str) -> bytes: + headers = {"Authorization": f"Bearer {api_key}"} + chunks = [] + word_count = 0 + + async with websockets.connect(WS_URL, additional_headers=headers) as ws: + # Single payload, then flush. No pacing needed for TTS input. + await ws.send(json.dumps({ + "text": text, + "voice_id": VOICE_ID, + "model": MODEL, + "sample_rate": SAMPLE_RATE, + "word_timestamps": True, + })) + await ws.send(json.dumps({"flush": True})) + + async for message in ws: + frame = json.loads(message) + status = frame.get("status") + + if status == "error": + detail = frame.get("data") or frame.get("message") or frame + print(f"Error: {detail}", file=sys.stderr) + sys.exit(1) + + if status == "chunk": + chunks.append(base64.b64decode(frame["data"]["audio"])) + + elif status == "word_timestamp": + payload = frame.get("data", {}) + entries = payload if isinstance(payload, list) else payload.get("words", [payload]) + for entry in entries: + print_word(entry) + word_count += 1 + + elif status == "complete": + break + + print(f"\n{word_count} words, {len(chunks)} audio chunks") + return b"".join(chunks) + + +def save_wav(pcm_data: bytes, path: str) -> None: + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) # 16-bit + f.setframerate(SAMPLE_RATE) + f.writeframes(pcm_data) + + +def main(): + api_key = os.environ.get("SMALLEST_API_KEY") + if not api_key: + print("Error: SMALLEST_API_KEY environment variable not set", file=sys.stderr) + sys.exit(1) + + text = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_TEXT + + print(f"Synthesizing with {MODEL} ({VOICE_ID})...\n") + pcm_data = asyncio.run(synthesize(text, api_key)) + + save_wav(pcm_data, OUTPUT_FILE) + print(f"Saved to {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/voice-agents/governance/app.py b/voice-agents/governance/app.py index c0dad20..4be9b45 100644 --- a/voice-agents/governance/app.py +++ b/voice-agents/governance/app.py @@ -1,100 +1,61 @@ -""" -Governed Voice Agent: PII Redaction & Cost Caps for Voice AI -============================================================ - -This example demonstrates how to add governance to a Smallest AI voice agent: -1. PII detection in transcribed speech (before it reaches the LLM) -2. Cost budget enforcement per call session -3. Tool authorization for voice-triggered actions -4. Structured audit trail for voice compliance (HIPAA, PCI-DSS) +"""Governed voice agent: TealTiger guardrails around a Smallest AI crew node. -The scenario: A customer support voice agent handles calls where callers -speak sensitive data (SSNs, credit card numbers, account numbers). Without -governance, this PII flows through the STT → LLM → TTS pipeline unscanned. +Every user turn is scanned before it reaches the LLM (PII is redacted in the +conversation context) and every LLM response is scanned before it is spoken. +A per-call turn cap ends runaway sessions gracefully. Requirements: pip install smallestai tealtiger Run: export SMALLEST_API_KEY="your-key" + export OPENAI_API_KEY="your-openai-key" python app.py """ -from smallestai.agentic import ( - AtomsCrewApp, - OutputCrewNode, - function_tool, - ToolRegistry, -) -from tealtiger import observe, TealEngine, PolicyMode +import os +from dotenv import load_dotenv +from loguru import logger -# ───────────────────────────────────────────────────────────────── -# Step 1: Configure TealTiger governance for voice -# ───────────────────────────────────────────────────────────────── - -engine = TealEngine( - policies=[ - # PII Detection: Scan transcribed speech for sensitive data - { - "type": "pii", - "action": "REDACT", - "patterns": ["ssn", "credit_card", "phone", "email", "account_number"], - # PII is redacted BEFORE it reaches the LLM - }, - - # Cost Governance: Cap per-call spend (voice = STT + LLM + TTS per turn) - { - "type": "cost_limit", - "max_per_session": 2.00, # $2 max per call - "action": "BLOCK", - }, - - # Tool Authorization: Only allow safe tools for voice agents - { - "type": "tool_allowlist", - "tools": ["lookup_account", "check_balance", "transfer_to_human"], - }, - - # Rate Limiting: Prevent runaway agent loops - { - "type": "rate_limit", - "max_calls": 20, - "window": "5m", - }, - ], - mode=PolicyMode.ENFORCE, -) +from smallestai.atoms.crew.clients.openai import OpenAIClient +from smallestai.atoms.crew.events import SDKEvent, SDKSystemUserJoinedEvent +from smallestai.atoms.crew.nodes import OutputCrewNode +from smallestai.atoms.crew.server import AtomsCrewApp +from smallestai.atoms.crew.session import CrewSession +from smallestai.atoms.crew.tools import ToolRegistry, function_tool +from tealtiger import GuardrailEngine, PIIDetectionGuardrail +load_dotenv() # ───────────────────────────────────────────────────────────────── -# Step 2: Define voice agent tools (with governance) +# Step 1: Configure TealTiger guardrails # ───────────────────────────────────────────────────────────────── -tool_registry = ToolRegistry() - +# One engine scans both directions: caller speech before the LLM sees it, +# and the LLM's reply before TTS speaks it. +guardrails = GuardrailEngine() +guardrails.register_guardrail(PIIDetectionGuardrail()) -@function_tool(tool_registry) -def lookup_account(account_id: str) -> str: - """Look up a customer account by ID.""" - # In production, this queries your database - return f"Account {account_id}: Premium tier, active since 2024" +MAX_TURNS_PER_CALL = 20 # runaway-loop cap -@function_tool(tool_registry) -def check_balance(account_id: str) -> str: - """Check account balance.""" - return f"Account {account_id} balance: $1,234.56" +async def redact(text: str) -> tuple[str, list[str]]: + """Scan text with the guardrail engine; return (redacted_text, pii_types).""" + result = await guardrails.execute(text) + if result.passed: + return text, [] - -@function_tool(tool_registry) -def transfer_to_human(reason: str) -> str: - """Transfer the call to a human agent.""" - return f"Transferring to human agent. Reason: {reason}" + found: list[str] = [] + for entry in result.results: + for detection in entry["result"].get("metadata", {}).get("detections", []): + found.append(detection["type"]) + text = text.replace(detection["value"], f"[{detection['type'].upper()} REDACTED]") + return text, found # ───────────────────────────────────────────────────────────────── -# Step 3: Create the governed voice agent +# Step 2: The governed agent node # ───────────────────────────────────────────────────────────────── SYSTEM_PROMPT = """You are a customer support agent for Acme Corp. @@ -102,74 +63,114 @@ def transfer_to_human(reason: str) -> str: IMPORTANT: - Never repeat sensitive information back to the caller -- If the caller provides their SSN or card number, acknowledge receipt +- If the caller provides an SSN or card number, acknowledge receipt without repeating it - For complex issues, transfer to a human agent """ class GovernedSupportAgent(OutputCrewNode): - """Voice agent with TealTiger governance at every turn.""" + """Voice agent with guardrails at every turn.""" - async def generate_response(self, transcript: str) -> str: - """Process caller speech with governance.""" + def __init__(self): + super().__init__(name="governed-support-agent") + self.llm = OpenAIClient( + model="gpt-4o-mini", + api_key=os.getenv("OPENAI_API_KEY"), + ) + self.turns = 0 - # Governance evaluates the transcript BEFORE it reaches the LLM - # If PII is detected, it's redacted in the transcript - # If cost budget is exceeded, the call is ended gracefully + self.tool_registry = ToolRegistry() + self.tool_registry.discover(self) + self.tool_schemas = self.tool_registry.get_schemas() - decision = engine.evaluate( - content=transcript, - context={ - "stage": "input", - "call_id": self.call_context.get("call_id", "unknown"), - }, - ) + self.context.add_message({"role": "system", "content": SYSTEM_PROMPT}) - if decision.action == "BLOCK": - # Budget exceeded or policy violation - return ( + async def generate_response(self): + """One governed turn: redact input, cap turns, scan output.""" + self.turns += 1 + if self.turns > MAX_TURNS_PER_CALL: + yield ( "I apologize, but I need to end this call. " "Please call back or visit our website for further assistance." ) + return + + # Redact PII in the caller's last message BEFORE the LLM sees it. + for message in reversed(self.context.messages): + if message["role"] == "user": + cleaned, pii = await redact(message["content"]) + if pii: + logger.warning(f"Redacted PII from caller turn: {pii}") + message["content"] = cleaned + break + + # Buffer the full response so it can be scanned before TTS speaks it. + response = await self.llm.chat(messages=self.context.messages, stream=True) + full_response = "" + async for chunk in response: + if chunk.content: + full_response += chunk.content + + cleaned, pii = await redact(full_response) + if pii: + logger.warning(f"Redacted PII from agent response: {pii}") + + if cleaned: + self.context.add_message({"role": "assistant", "content": cleaned}) + yield cleaned + + @function_tool() + def lookup_account(self, account_id: str) -> str: + """Look up a customer account by ID. + + Args: + account_id: The account identifier. + """ + return f"Account {account_id}: Premium tier, active since 2024" + + @function_tool() + def check_balance(self, account_id: str) -> str: + """Check account balance. + + Args: + account_id: The account identifier. + """ + return f"Account {account_id} balance: $1,234.56" + + @function_tool() + def transfer_to_human(self, reason: str) -> str: + """Transfer the call to a human agent. + + Args: + reason: Why the caller needs a human. + """ + return f"Transferring to human agent. Reason: {reason}" - if decision.action == "REDACT": - # PII was detected and redacted — use cleaned transcript - transcript = decision.redacted_content - - # Now the LLM only sees redacted content - response = await self.llm.generate( - system_prompt=SYSTEM_PROMPT, - user_message=transcript, - tools=tool_registry, - ) - # Scan the response before TTS speaks it - output_decision = engine.evaluate( - content=response, - context={"stage": "output"}, - ) +# ───────────────────────────────────────────────────────────────── +# Step 3: Run the voice agent +# ───────────────────────────────────────────────────────────────── - if output_decision.action == "REDACT": - response = output_decision.redacted_content - return response +async def setup_session(session: CrewSession): + agent = GovernedSupportAgent() + session.add_node(agent) + await session.start() + @session.on_event("on_event_received") + async def on_event_received(_, event: SDKEvent): + if isinstance(event, SDKSystemUserJoinedEvent): + greeting = "Hello! You've reached Acme Corp support. How can I help you today?" + agent.context.add_message({"role": "assistant", "content": greeting}) + await agent.speak(greeting) -# ───────────────────────────────────────────────────────────────── -# Step 4: Run the voice agent -# ───────────────────────────────────────────────────────────────── + await session.wait_until_complete() + logger.success("Session complete") -app = AtomsCrewApp( - agent=GovernedSupportAgent( - model="gpt-4o-mini", - voice="sophia", - language="en", - ), -) if __name__ == "__main__": print("Starting governed voice agent...") - print("Governance: PII redaction, $2/call budget, tool allowlisting") - print("Mode: ENFORCE (violations are blocked)") + print(f"Guardrails: PII redaction on input and output, {MAX_TURNS_PER_CALL}-turn cap per call") + app = AtomsCrewApp(setup_handler=setup_session) app.run() diff --git a/voice-agents/ios_swift_voice_agent/README.md b/voice-agents/ios_swift_voice_agent/README.md index cfbc4e1..687c64f 100644 --- a/voice-agents/ios_swift_voice_agent/README.md +++ b/voice-agents/ios_swift_voice_agent/README.md @@ -2,7 +2,7 @@ A minimal native iOS sample that opens a real-time voice session with a Smallest AI voice agent over the plain WebSocket endpoint. SwiftUI, `URLSessionWebSocketTask`, `AVAudioEngine`. iOS 16+. -The app has the same UX as the [React Native cookbook](../react_native_voice_agent/) — status chip, two labelled waveforms (narrator + you), mute toggle, sending counter, in-app settings sheet wired to the full `draft → publish → activate` REST flow. +The app has the same UX as the [React Native cookbook](../react_native_voice_agent/) — status chip, two labelled waveforms (narrator + you), mute toggle, sending counter, in-app settings sheet wired to the full branch `draft → publish` REST flow. ## What it shows @@ -11,7 +11,7 @@ The app has the same UX as the [React Native cookbook](../react_native_voice_age - Gapless playback of `output_audio.delta` chunks via `AVAudioPlayerNode.scheduleBuffer`. - Full protocol: `input_audio_buffer.append` streaming, `agent_start_talking` / `agent_stop_talking` / `interruption` / `session.closed` / `error` handling. - Exponential-backoff reconnect on transient network errors; hard-stop on auth failures. -- In-app settings sheet: voice / speed / language pickers that drive the five-step `draft → publish → activate` REST dance. +- In-app settings sheet: voice / speed / language pickers that drive the branch `draft → publish` REST dance. - Mute toggle gates the mic-upload path client-side — useful during narration when room noise is tripping server VAD. - Transport diagnostics: a live "sending · N" counter under the you waveform, independent of mic level. @@ -48,7 +48,7 @@ In Xcode: | Layer | File | Responsibility | |---|---|---| | Transport | `Sources/Clients/AtomsClient.swift` | `URLSessionWebSocketTask` wrapper with exponential-backoff reconnect, event dispatch, auth-close hard-stop. | -| REST | `Sources/Clients/AtomsRest.swift` | Thin `URLSession` wrapper for the `draft → publish → activate` flow used by the settings sheet. | +| REST | `Sources/Clients/AtomsRest.swift` | Thin `URLSession` wrapper for the branch `draft → publish` flow used by the settings sheet. | | Audio | `Sources/Audio/AudioEngine.swift` | `AVAudioEngine` setup (`.playAndRecord` + `.voiceChat` + `defaultToSpeaker`), mic tap with inline resample to Int16 @ 24 kHz, `AVAudioPlayerNode` for gapless playback. | | State machine | `Sources/ViewModels/SessionViewModel.swift` | `@MainActor ObservableObject`. Owns lifecycle, permission flow, mute gating, mic-chunk counter, error classification. | | UI | `Sources/Views/*.swift` | SwiftUI single-screen app — title card, status chip, two labelled waveforms, mute pill, send counter, settings sheet, call button, error banner. | @@ -71,7 +71,7 @@ Tap **settings** (top-right, idle screen) to open the agent configuration sheet: - **Speed** — 0.85× / 1.00× / 1.15× / 1.30×. - **Language** — English, Hindi, Multi (auto-detect). -**Apply & publish** runs the five-step REST flow (`GET /versions` → `POST /drafts` → `PATCH /drafts/.../config` → `POST /drafts/.../publish` → `PATCH /versions/.../activate`) against your live agent. End the current session and start a new one to hear the change. +**Apply & publish** runs the branch REST flow (`GET /branches` → `PUT /branches/.../draft` → `POST /branches/.../draft/publish`, which makes the new revision live) against your live agent. End the current session and start a new one to hear the change. ## Testing diff --git a/voice-agents/ios_swift_voice_agent/Sources/Clients/AtomsRest.swift b/voice-agents/ios_swift_voice_agent/Sources/Clients/AtomsRest.swift index 76eec21..5c1a039 100644 --- a/voice-agents/ios_swift_voice_agent/Sources/Clients/AtomsRest.swift +++ b/voice-agents/ios_swift_voice_agent/Sources/Clients/AtomsRest.swift @@ -1,13 +1,11 @@ import Foundation /// Thin wrapper around the Atoms REST surface the app needs to update a live -/// agent's voice / speed / language. Full dance: -/// 1. GET /agent/{id} read current -/// 2. GET /agent/{id}/versions?limit=1 find source version -/// 3. POST /agent/{id}/drafts open draft -/// 4. PATCH /agent/{id}/drafts/{d}/config write new values -/// 5. POST /agent/{id}/drafts/{d}/publish publish as new version -/// 6. PATCH /agent/{id}/versions/{v}/activate make it live +/// agent's voice / speed / language. Full dance (v2 branches flow): +/// 1. GET /agent/{id} read current +/// 2. GET /agent/{id}/branches find the live branch +/// 3. PUT /agent/{id}/branches/{b}/draft write new values into the open draft +/// 4. POST /agent/{id}/branches/{b}/draft/publish publish; the revision goes live enum AtomsRest { private static let base = "https://api.smallest.ai/atoms/v1" @@ -30,9 +28,8 @@ enum AtomsRest { enum RestError: Error { case http(status: Int, body: String) case parse - case noSourceVersion - case noDraftId - case noVersionId + case noBranch + case noRevisionId } static func fetchAgent(apiKey: String, agentId: String) async throws -> AgentSnapshot { @@ -54,27 +51,20 @@ enum AtomsRest { @discardableResult static func updateAgentConfig(apiKey: String, agentId: String, current: AgentSnapshot, patch: UpdateInput) async throws -> String { - let versionsJson = try await request(method: "GET", - path: "/agent/\(agentId)/versions?limit=1", + let branchesJson = try await request(method: "GET", + path: "/agent/\(agentId)/branches", apiKey: apiKey) - let versionsData = (versionsJson["data"] as? [String: Any]) ?? versionsJson - let versions = (versionsData["versions"] as? [[String: Any]]) ?? [] - guard let sourceVersion = versions.first?["_id"] as? String else { - throw RestError.noSourceVersion + let branchesData = (branchesJson["data"] as? [String: Any]) ?? branchesJson + let branches = (branchesData["branches"] as? [[String: Any]]) ?? [] + // Pick the live branch; fall back to the default branch, then the first. + let entry = branches.first { (($0["isLive"] ?? $0["is_live"]) as? Bool) == true } + ?? branches.first { ((($0["branch"] as? [String: Any])?["isDefault"] ?? ($0["branch"] as? [String: Any])?["is_default"]) as? Bool) == true } + ?? branches.first + let branchObj = entry?["branch"] as? [String: Any] + guard let branchId = (branchObj?["_id"] ?? branchObj?["id"]) as? String else { + throw RestError.noBranch } - let draftJson = try await request( - method: "POST", - path: "/agent/\(agentId)/drafts", - apiKey: apiKey, - body: [ - "draftName": "live-config-\(Int(Date().timeIntervalSince1970))", - "sourceVersionId": sourceVersion, - ] - ) - let draftData = (draftJson["data"] as? [String: Any]) ?? draftJson - guard let draftId = draftData["draftId"] as? String else { throw RestError.noDraftId } - let nextVoiceId = patch.voiceId ?? current.voiceId let nextVoiceModel = patch.voiceModel ?? current.voiceModel let nextSpeed = patch.speed ?? current.speed @@ -94,24 +84,39 @@ enum AtomsRest { "speed": nextSpeed, ], ] - _ = try await request(method: "PATCH", - path: "/agent/\(agentId)/drafts/\(draftId)/config", + // PUT creates the branch's open draft when there is none, otherwise + // updates it in place. + _ = try await request(method: "PUT", + path: "/agent/\(agentId)/branches/\(branchId)/draft", apiKey: apiKey, body: configBody) + // Publishing the draft makes the new revision live; no activate step in v2. let publishJson = try await request( method: "POST", - path: "/agent/\(agentId)/drafts/\(draftId)/publish", + path: "/agent/\(agentId)/branches/\(branchId)/draft/publish", apiKey: apiKey, body: ["label": "ios-\(Int(Date().timeIntervalSince1970))"] ) let publishData = (publishJson["data"] as? [String: Any]) ?? publishJson - guard let newVersion = publishData["_id"] as? String else { throw RestError.noVersionId } - - _ = try await request(method: "PATCH", - path: "/agent/\(agentId)/versions/\(newVersion)/activate", - apiKey: apiKey) - return newVersion + if let newRevision = (publishData["_id"] as? String) ?? (publishData["id"] as? String) { + return newRevision + } + // Publishing runs an async security scan; poll until the draft closes. + if publishData["state"] != nil { + for _ in 0..<60 { + try await Task.sleep(nanoseconds: 2_000_000_000) + let branchJson = try await request(method: "GET", + path: "/agent/\(agentId)/branches/\(branchId)", + apiKey: apiKey) + let data = (branchJson["data"] as? [String: Any]) ?? branchJson + let branch = (data["branch"] as? [String: Any]) ?? data + if branch["openDraftId"] == nil || branch["openDraftId"] is NSNull { + return (branch["headRevisionId"] as? String) ?? "published" + } + } + } + throw RestError.noRevisionId } // MARK: - Private diff --git a/voice-agents/ios_swift_voice_agent/Sources/Views/SettingsSheet.swift b/voice-agents/ios_swift_voice_agent/Sources/Views/SettingsSheet.swift index 3a18daf..bdf36ec 100644 --- a/voice-agents/ios_swift_voice_agent/Sources/Views/SettingsSheet.swift +++ b/voice-agents/ios_swift_voice_agent/Sources/Views/SettingsSheet.swift @@ -1,7 +1,7 @@ import SwiftUI /// Agent config picker. Mirrors the React Native cookbook's settings sheet. -/// Runs the full 5-step draft → publish → activate REST flow on Apply. +/// Runs the full branch draft → publish REST flow on Apply. struct SettingsSheet: View { @Binding var isPresented: Bool let apiKey: String diff --git a/voice-agents/react_native_voice_agent/README.md b/voice-agents/react_native_voice_agent/README.md index 30e17c3..c657898 100644 --- a/voice-agents/react_native_voice_agent/README.md +++ b/voice-agents/react_native_voice_agent/README.md @@ -12,7 +12,7 @@ The app is deliberately small so it reads as a reference for anyone wiring a voi - Microphone capture and gapless PCM playback using [`react-native-audio-api`](https://docs.swmansion.com/react-native-audio-api/). - Full protocol: `input_audio_buffer.append` streaming, `output_audio.delta` scheduled playback, `agent_start_talking` / `agent_stop_talking` / `interruption` / `session.closed` handling. - Exponential-backoff reconnect for transient drops, hard-stop on auth failures. -- Programmatic agent creation and in-app reconfiguration via REST — voice, speed, and language pickers that drive the full `draft → publish → activate` flow. +- Programmatic agent creation and in-app reconfiguration via REST — voice, speed, and language pickers that drive the full branch `draft → publish` flow. - Transport diagnostics in the UI: a live chunk counter so users can verify their voice is actually reaching the server. - Mute toggle to suppress mic uploads during narration (stops spurious server-side VAD interruptions). - Correctly configured iOS audio session: `playAndRecord` + `default` mode + `defaultToSpeaker`, so narration plays through the loud bottom speaker at full media volume, cleanly, without distortion or buffer underruns. @@ -41,7 +41,7 @@ npm install npx expo prebuild --clean # regenerates ios/ and android/ projects ``` -`scripts/setup_agent.py` walks the full REST flow behind the scenes: `POST /agent` → open a draft → `PATCH /drafts/.../config` with the prompt, voice, and LLM model → `POST /drafts/.../publish` → `PATCH /versions/.../activate`. It is idempotent — re-running updates the existing agent in place instead of creating duplicates. +`scripts/setup_agent.py` walks the full REST flow behind the scenes: `POST /agent` → `GET /branches` to find the live branch → `PUT /branches/.../draft` with the prompt, voice, and LLM model → `POST /branches/.../draft/publish` (the published revision goes live on its own). It is idempotent — re-running updates the existing agent in place instead of creating duplicates. Flags: @@ -92,7 +92,7 @@ Tap **settings** (top-right, idle screen only) to open the agent configuration s - **Speed** — 0.85× / 1.00× / 1.15× / 1.30×. - **Language** — English, Hindi, or Multi (auto-detect). -**Apply & publish** runs the five-step REST flow (open draft → PATCH config → publish version → activate version) against your live agent. End the current story and start a new one to hear the change. +**Apply & publish** runs the branch REST flow (find the live branch → PUT the draft config → publish the draft, which makes the new revision live) against your live agent. End the current story and start a new one to hear the change. ### During a session @@ -104,7 +104,7 @@ Tap **settings** (top-right, idle screen only) to open the agent configuration s | Layer | Module | Responsibility | |---|---|---| | Transport | `src/agent/AtomsClient.ts` | Opens the WebSocket, dispatches server events, handles reconnect with backoff. | -| REST | `src/agent/atomsRest.ts` | Thin `fetch` wrapper for the agent read + draft-publish-activate flow used by the settings sheet. | +| REST | `src/agent/atomsRest.ts` | Thin `fetch` wrapper for the agent read + branch draft-publish flow used by the settings sheet. | | Capture | `src/agent/audioCapture.ts` | Configures the iOS audio session (`playAndRecord` + `default` + `defaultToSpeaker`), starts an `AudioRecorder`, converts Float32 → Int16 LE, emits RMS for the mic waveform. | | Playback | `src/agent/audioPlayback.ts` | Web Audio `AudioContext` with a `nextPlayTime` pointer for gapless scheduling; `ctx.resume()` after construction (Android starts suspended); `flush()` resets the pointer on `interruption`. | | State machine | `src/hooks/useAtomsSession.ts` | `idle → connecting → joined → listening → narrating → error`. Owns permission check, lifecycle, mute gating, mic-chunk counter, error classification. | diff --git a/voice-agents/react_native_voice_agent/scripts/setup_agent.py b/voice-agents/react_native_voice_agent/scripts/setup_agent.py index edda34a..143f095 100644 --- a/voice-agents/react_native_voice_agent/scripts/setup_agent.py +++ b/voice-agents/react_native_voice_agent/scripts/setup_agent.py @@ -10,10 +10,11 @@ 2. Create-or-update — If AGENT_ID is missing, requires SMALLEST_API_KEY. Looks up an existing agent by name; if found, updates its config in - place. Otherwise creates a fresh agent. In both cases: opens a draft, - writes the full single-prompt config (prompt, voice, language, - model), and publishes the draft as a new version. Writes AGENT_ID - (and EXPO_PUBLIC_* mirrors for Metro bundle inlining) into .env. + place. Otherwise creates a fresh agent. In both cases: writes the + full single-prompt config (prompt, voice, language, model) into the + live branch's draft and publishes it, which makes the new revision + live. Writes AGENT_ID (and EXPO_PUBLIC_* mirrors for Metro bundle + inlining) into .env. Usage: cd voice-agents/atoms_hearthside_rn @@ -215,45 +216,33 @@ def create_agent(api_key: str, name: str, slm: str, language: str, return agent_id -def fetch_published_version_id(api_key: str, agent_id: str) -> Optional[str]: - resp = api_request("GET", f"/agent/{agent_id}/versions?limit=1", api_key) +def fetch_branch_id(api_key: str, agent_id: str) -> str: + # v2 versioning: every agent carries a set of branches. The live branch + # holds the deployed config; fall back to the default branch when + # nothing is live yet (fresh agents). + resp = api_request("GET", f"/agent/{agent_id}/branches", api_key) data = unwrap_data(resp) - items = data.get("versions") if isinstance(data, dict) else data - if isinstance(items, list) and items: - v = items[0] - return v.get("_id") or v.get("id") or v.get("versionId") - return None - - -def fetch_or_create_draft(api_key: str, agent_id: str, source_version_id: Optional[str]) -> str: - # AgentVersion has two id fields: _id (mongo record id) and draftId - # (the routing identifier used on PATCH/PUBLISH paths). We must use - # draftId for API routes, not _id. - try: - resp = api_request("GET", f"/agent/{agent_id}/drafts", api_key) - drafts = unwrap_data(resp) - if isinstance(drafts, list) and drafts: - d = drafts[0] - draft_id = d.get("draftId") - if draft_id: - return draft_id - except ApiError: - pass - - body: dict[str, Any] = {"draftName": "hearthside-setup"} - if source_version_id: - body["sourceVersionId"] = source_version_id - resp = api_request("POST", f"/agent/{agent_id}/drafts", api_key, body) - data = unwrap_data(resp) - draft_id = data.get("draftId") if isinstance(data, dict) else None - if not draft_id: - raise SystemExit(f"Unexpected /drafts response (no draftId): {json.dumps(resp)[:400]}") - return draft_id - - -def patch_draft_config(api_key: str, agent_id: str, draft_id: str, *, - slm: str, language: str, prompt: str, - voice_id: Optional[str], voice_model: Optional[str]) -> None: + branches = data.get("branches") if isinstance(data, dict) else None + if not isinstance(branches, list) or not branches: + raise SystemExit(f"No branches on agent {agent_id}: {json.dumps(resp)[:400]}") + entry = next((b for b in branches if isinstance(b, dict) and (b.get("isLive") or b.get("is_live"))), None) + if entry is None: + entry = next((b for b in branches + if isinstance(b, dict) and ((b.get("branch") or {}).get("isDefault") or (b.get("branch") or {}).get("is_default"))), None) + if entry is None: + entry = branches[0] + branch_obj = (entry.get("branch") or {}) if isinstance(entry, dict) else {} + branch_id = branch_obj.get("_id") or branch_obj.get("id") + if not branch_id: + raise SystemExit(f"Branch entry has no id: {json.dumps(entry)[:400]}") + return branch_id + + +def edit_branch_draft(api_key: str, agent_id: str, branch_id: str, *, + slm: str, language: str, prompt: str, + voice_id: Optional[str], voice_model: Optional[str]) -> None: + # PUT creates the branch's open draft when there is none, otherwise + # updates it in place. Field names match the old draft-config payload. body: dict[str, Any] = { "language": { "default": language, @@ -274,22 +263,28 @@ def patch_draft_config(api_key: str, agent_id: str, draft_id: str, *, }, "speed": 1.0, } - api_request("PATCH", f"/agent/{agent_id}/drafts/{draft_id}/config", api_key, body) + api_request("PUT", f"/agent/{agent_id}/branches/{branch_id}/draft", api_key, body) -def publish_draft(api_key: str, agent_id: str, draft_id: str) -> str: - """Publish a draft; returns the new version id (needed for activation).""" - resp = api_request("POST", f"/agent/{agent_id}/drafts/{draft_id}/publish", api_key, +def publish_branch_draft(api_key: str, agent_id: str, branch_id: str) -> str: + """Publish the branch's open draft; the published revision goes live + on its own (no separate activate step in v2).""" + resp = api_request("POST", f"/agent/{agent_id}/branches/{branch_id}/draft/publish", api_key, {"label": f"hearthside-{int(time.time())}"}) data = unwrap_data(resp) - version_id = data.get("_id") if isinstance(data, dict) else None - if not version_id: - raise SystemExit(f"Publish did not return a version id: {json.dumps(resp)[:400]}") - return version_id - - -def activate_version(api_key: str, agent_id: str, version_id: str) -> None: - api_request("PATCH", f"/agent/{agent_id}/versions/{version_id}/activate", api_key) + revision_id = (data.get("_id") or data.get("id")) if isinstance(data, dict) else None + if revision_id: + return revision_id + # Publishing runs an async security scan; poll until the draft closes. + if isinstance(data, dict) and data.get("state"): + for _ in range(60): + time.sleep(2) + b = unwrap_data(api_request("GET", f"/agent/{agent_id}/branches/{branch_id}", api_key)) + branch = b.get("branch", b) if isinstance(b, dict) else {} + if not branch.get("openDraftId"): + return branch.get("headRevisionId") or "published" + raise SystemExit("Publish did not finish scanning within 120s") + raise SystemExit(f"Publish did not return a revision id: {json.dumps(resp)[:400]}") def verify_agent_exists(api_key: str, agent_id: str) -> bool: @@ -354,24 +349,19 @@ def main() -> int: args.voice, args.voice_model) print(f" created agent: {agent_id}") - print("Opening draft for config edit...") - version_id = fetch_published_version_id(api_key, agent_id) - draft_id = fetch_or_create_draft(api_key, agent_id, version_id) - print(f" draft: {draft_id}") + print("Resolving live branch...") + branch_id = fetch_branch_id(api_key, agent_id) + print(f" branch: {branch_id}") print("Writing prompt, LLM, and language into draft...") - patch_draft_config(api_key, agent_id, draft_id, - slm=args.model, language=args.language, - prompt=NARRATOR_PROMPT, - voice_id=args.voice, voice_model=args.voice_model) + edit_branch_draft(api_key, agent_id, branch_id, + slm=args.model, language=args.language, + prompt=NARRATOR_PROMPT, + voice_id=args.voice, voice_model=args.voice_model) print("Publishing draft...") - new_version_id = publish_draft(api_key, agent_id, draft_id) - print(f" published as version {new_version_id}.") - - print("Activating new version...") - activate_version(api_key, agent_id, new_version_id) - print(" activated (new config is live).") + revision_id = publish_branch_draft(api_key, agent_id, branch_id) + print(f" published as revision {revision_id} (new config is live).") _mirror_env_for_expo(api_key, agent_id) print(f"\nDone. Agent ID -> {agent_id}") diff --git a/voice-agents/react_native_voice_agent/src/agent/atomsRest.ts b/voice-agents/react_native_voice_agent/src/agent/atomsRest.ts index ae20273..c24c5c9 100644 --- a/voice-agents/react_native_voice_agent/src/agent/atomsRest.ts +++ b/voice-agents/react_native_voice_agent/src/agent/atomsRest.ts @@ -1,11 +1,9 @@ // Thin wrapper around the Atoms REST surface the app needs for updating a -// live agent's voice/speed/language. Full dance: -// 1. GET /agent/{id} (read current config) -// 2. GET /agent/{id}/versions?limit=1 (find version to branch from) -// 3. POST /agent/{id}/drafts (open a draft) -// 4. PATCH /agent/{id}/drafts/{d}/config (write new values) -// 5. POST /agent/{id}/drafts/{d}/publish (publish as new version) -// 6. PATCH /agent/{id}/versions/{v}/activate (make it live) +// live agent's voice/speed/language. Full dance (v2 branches flow): +// 1. GET /agent/{id} (read current config) +// 2. GET /agent/{id}/branches (find the live branch) +// 3. PUT /agent/{id}/branches/{b}/draft (write new values into the open draft) +// 4. POST /agent/{id}/branches/{b}/draft/publish (publish; the revision goes live) // Anything that doesn't change is carried forward from the current config. const API_BASE = 'https://api.smallest.ai/atoms/v1'; @@ -70,27 +68,23 @@ export interface UpdateInput { language?: string; } -// Runs the 5-step draft-publish-activate flow. Returns the new version id. +// Runs the branch edit-and-publish flow. Returns the published revision id. export async function updateAgentConfig( apiKey: string, agentId: string, current: AgentSnapshot, patch: UpdateInput, ): Promise { - const versionsResp = unwrap( - await call(apiKey, 'GET', `/agent/${agentId}/versions?limit=1`), + const branchesResp = unwrap( + await call(apiKey, 'GET', `/agent/${agentId}/branches`), ); - const sourceVersion = (versionsResp?.versions ?? [])[0]?._id; - if (!sourceVersion) throw new Error('No source version found on agent'); - - const draftResp = unwrap( - await call(apiKey, 'POST', `/agent/${agentId}/drafts`, { - draftName: `live-config-${Date.now()}`, - sourceVersionId: sourceVersion, - }), - ); - const draftId: string = draftResp.draftId; - if (!draftId) throw new Error('Draft creation did not return draftId'); + const branches: any[] = branchesResp?.branches ?? []; + const entry = + branches.find((b) => b?.isLive ?? b?.is_live) ?? + branches.find((b) => b?.branch?.isDefault ?? b?.branch?.is_default) ?? + branches[0]; + const branchId: string = entry?.branch?._id ?? entry?.branch?.id; + if (!branchId) throw new Error('No branch found on agent'); const nextVoiceId = patch.voiceId ?? current.voiceId; const nextVoiceModel = patch.voiceModel ?? current.voiceModel; @@ -111,17 +105,27 @@ export async function updateAgentConfig( }, }; - await call(apiKey, 'PATCH', `/agent/${agentId}/drafts/${draftId}/config`, configBody); + // PUT creates the branch's open draft when there is none, otherwise + // updates it in place. + await call(apiKey, 'PUT', `/agent/${agentId}/branches/${branchId}/draft`, configBody); + // Publishing the draft makes the new revision live; no activate step in v2. const publishResp = unwrap( - await call(apiKey, 'POST', `/agent/${agentId}/drafts/${draftId}/publish`, { + await call(apiKey, 'POST', `/agent/${agentId}/branches/${branchId}/draft/publish`, { label: `hearthside-${Date.now()}`, }), ); - const newVersion: string = publishResp._id; - if (!newVersion) throw new Error('Publish did not return version id'); - - await call(apiKey, 'PATCH', `/agent/${agentId}/versions/${newVersion}/activate`); + let newRevision: string | undefined = publishResp?._id ?? publishResp?.id; + if (!newRevision && publishResp?.state) { + // Publishing runs an async security scan; poll until the draft closes. + for (let i = 0; i < 60 && !newRevision; i++) { + await new Promise((r) => setTimeout(r, 2000)); + const b = unwrap(await call(apiKey, 'GET', `/agent/${agentId}/branches/${branchId}`)); + const branch = b?.branch ?? b; + if (!branch?.openDraftId) newRevision = branch?.headRevisionId ?? 'published'; + } + } + if (!newRevision) throw new Error('Publish did not return revision id'); - return newVersion; + return newRevision; } diff --git a/voice-agents/react_native_voice_widget/scripts/setup_agent.py b/voice-agents/react_native_voice_widget/scripts/setup_agent.py index b0bef3c..fb8aa91 100644 --- a/voice-agents/react_native_voice_widget/scripts/setup_agent.py +++ b/voice-agents/react_native_voice_widget/scripts/setup_agent.py @@ -10,10 +10,11 @@ 2. Create-or-update — If AGENT_ID is missing, requires SMALLEST_API_KEY. Looks up an existing agent by name; if found, updates its config in - place. Otherwise creates a fresh agent. In both cases: opens a draft, - writes the full single-prompt config (prompt, voice, language, - model), and publishes the draft as a new version. Writes AGENT_ID - (and EXPO_PUBLIC_* mirrors for Metro bundle inlining) into .env. + place. Otherwise creates a fresh agent. In both cases: writes the + full single-prompt config (prompt, voice, language, model) into the + live branch's draft and publishes it, which makes the new revision + live. Writes AGENT_ID (and EXPO_PUBLIC_* mirrors for Metro bundle + inlining) into .env. Usage: cd voice-agents/atoms_hearthside_rn @@ -222,45 +223,33 @@ def create_agent(api_key: str, name: str, slm: str, language: str, return agent_id -def fetch_published_version_id(api_key: str, agent_id: str) -> Optional[str]: - resp = api_request("GET", f"/agent/{agent_id}/versions?limit=1", api_key) +def fetch_branch_id(api_key: str, agent_id: str) -> str: + # v2 versioning: every agent carries a set of branches. The live branch + # holds the deployed config; fall back to the default branch when + # nothing is live yet (fresh agents). + resp = api_request("GET", f"/agent/{agent_id}/branches", api_key) data = unwrap_data(resp) - items = data.get("versions") if isinstance(data, dict) else data - if isinstance(items, list) and items: - v = items[0] - return v.get("_id") or v.get("id") or v.get("versionId") - return None - - -def fetch_or_create_draft(api_key: str, agent_id: str, source_version_id: Optional[str]) -> str: - # AgentVersion has two id fields: _id (mongo record id) and draftId - # (the routing identifier used on PATCH/PUBLISH paths). We must use - # draftId for API routes, not _id. - try: - resp = api_request("GET", f"/agent/{agent_id}/drafts", api_key) - drafts = unwrap_data(resp) - if isinstance(drafts, list) and drafts: - d = drafts[0] - draft_id = d.get("draftId") - if draft_id: - return draft_id - except ApiError: - pass - - body: dict[str, Any] = {"draftName": "hearthside-setup"} - if source_version_id: - body["sourceVersionId"] = source_version_id - resp = api_request("POST", f"/agent/{agent_id}/drafts", api_key, body) - data = unwrap_data(resp) - draft_id = data.get("draftId") if isinstance(data, dict) else None - if not draft_id: - raise SystemExit(f"Unexpected /drafts response (no draftId): {json.dumps(resp)[:400]}") - return draft_id - - -def patch_draft_config(api_key: str, agent_id: str, draft_id: str, *, - slm: str, language: str, prompt: str, - voice_id: Optional[str], voice_model: Optional[str]) -> None: + branches = data.get("branches") if isinstance(data, dict) else None + if not isinstance(branches, list) or not branches: + raise SystemExit(f"No branches on agent {agent_id}: {json.dumps(resp)[:400]}") + entry = next((b for b in branches if isinstance(b, dict) and (b.get("isLive") or b.get("is_live"))), None) + if entry is None: + entry = next((b for b in branches + if isinstance(b, dict) and ((b.get("branch") or {}).get("isDefault") or (b.get("branch") or {}).get("is_default"))), None) + if entry is None: + entry = branches[0] + branch_obj = (entry.get("branch") or {}) if isinstance(entry, dict) else {} + branch_id = branch_obj.get("_id") or branch_obj.get("id") + if not branch_id: + raise SystemExit(f"Branch entry has no id: {json.dumps(entry)[:400]}") + return branch_id + + +def edit_branch_draft(api_key: str, agent_id: str, branch_id: str, *, + slm: str, language: str, prompt: str, + voice_id: Optional[str], voice_model: Optional[str]) -> None: + # PUT creates the branch's open draft when there is none, otherwise + # updates it in place. Field names match the old draft-config payload. body: dict[str, Any] = { "language": { "default": language, @@ -281,22 +270,28 @@ def patch_draft_config(api_key: str, agent_id: str, draft_id: str, *, }, "speed": 1.0, } - api_request("PATCH", f"/agent/{agent_id}/drafts/{draft_id}/config", api_key, body) + api_request("PUT", f"/agent/{agent_id}/branches/{branch_id}/draft", api_key, body) -def publish_draft(api_key: str, agent_id: str, draft_id: str) -> str: - """Publish a draft; returns the new version id (needed for activation).""" - resp = api_request("POST", f"/agent/{agent_id}/drafts/{draft_id}/publish", api_key, +def publish_branch_draft(api_key: str, agent_id: str, branch_id: str) -> str: + """Publish the branch's open draft; the published revision goes live + on its own (no separate activate step in v2).""" + resp = api_request("POST", f"/agent/{agent_id}/branches/{branch_id}/draft/publish", api_key, {"label": f"hearthside-{int(time.time())}"}) data = unwrap_data(resp) - version_id = data.get("_id") if isinstance(data, dict) else None - if not version_id: - raise SystemExit(f"Publish did not return a version id: {json.dumps(resp)[:400]}") - return version_id - - -def activate_version(api_key: str, agent_id: str, version_id: str) -> None: - api_request("PATCH", f"/agent/{agent_id}/versions/{version_id}/activate", api_key) + revision_id = (data.get("_id") or data.get("id")) if isinstance(data, dict) else None + if revision_id: + return revision_id + # Publishing runs an async security scan; poll until the draft closes. + if isinstance(data, dict) and data.get("state"): + for _ in range(60): + time.sleep(2) + b = unwrap_data(api_request("GET", f"/agent/{agent_id}/branches/{branch_id}", api_key)) + branch = b.get("branch", b) if isinstance(b, dict) else {} + if not branch.get("openDraftId"): + return branch.get("headRevisionId") or "published" + raise SystemExit("Publish did not finish scanning within 120s") + raise SystemExit(f"Publish did not return a revision id: {json.dumps(resp)[:400]}") def verify_agent_exists(api_key: str, agent_id: str) -> bool: @@ -361,24 +356,19 @@ def main() -> int: args.voice, args.voice_model) print(f" created agent: {agent_id}") - print("Opening draft for config edit...") - version_id = fetch_published_version_id(api_key, agent_id) - draft_id = fetch_or_create_draft(api_key, agent_id, version_id) - print(f" draft: {draft_id}") + print("Resolving live branch...") + branch_id = fetch_branch_id(api_key, agent_id) + print(f" branch: {branch_id}") print("Writing prompt, LLM, and language into draft...") - patch_draft_config(api_key, agent_id, draft_id, - slm=args.model, language=args.language, - prompt=NARRATOR_PROMPT, - voice_id=args.voice, voice_model=args.voice_model) + edit_branch_draft(api_key, agent_id, branch_id, + slm=args.model, language=args.language, + prompt=NARRATOR_PROMPT, + voice_id=args.voice, voice_model=args.voice_model) print("Publishing draft...") - new_version_id = publish_draft(api_key, agent_id, draft_id) - print(f" published as version {new_version_id}.") - - print("Activating new version...") - activate_version(api_key, agent_id, new_version_id) - print(" activated (new config is live).") + revision_id = publish_branch_draft(api_key, agent_id, branch_id) + print(f" published as revision {revision_id} (new config is live).") _mirror_env_for_expo(api_key, agent_id) print(f"\nDone. Agent ID -> {agent_id}") diff --git a/voice-agents/web_call_session/.env.sample b/voice-agents/web_call_session/.env.sample new file mode 100644 index 0000000..dacd32c --- /dev/null +++ b/voice-agents/web_call_session/.env.sample @@ -0,0 +1,6 @@ +# Smallest AI API Key +# Get yours at https://smallest.ai/console +SMALLEST_API_KEY=your-smallest-api-key-here + +# Atoms agent to start sessions for (or pass --agent-id) +AGENT_ID=your-agent-id-here diff --git a/voice-agents/web_call_session/README.md b/voice-agents/web_call_session/README.md new file mode 100644 index 0000000..eccc9ce --- /dev/null +++ b/voice-agents/web_call_session/README.md @@ -0,0 +1,71 @@ +# Web Call Session + +Start a browser voice (or text chat) session for an Atoms agent from your server. Your backend calls the API with the secret key and hands the browser only what it needs to join the room. The API key never reaches the client. + +## Try It + +```bash +uv pip install -r requirements.txt + +python start_session.py --agent-id + +# Or a text chat session +python start_session.py --agent-id --chat +``` + +Output: + +``` +Starting web call session... + token: eyJhbGciOi... + room_name: 0f6c1c1e-9b1a-4d5e-8f2a-... + host: wss://...livekit.cloud + conversation_id: 68b1f0c2e4a1b2c3d4e5f6a7 + call_id: 68b1f0c2e4a1b2c3d4e5f6a8 + +After the call, fetch details with: + client.atoms.calls.get(id="68b1f0c2e4a1b2c3d4e5f6a7") +``` + +## Requirements + +> Base dependencies are installed via the root `requirements.txt`, plus `smallestai>=5.12.0` from the local `requirements.txt`. Add `SMALLEST_API_KEY` (and optionally `AGENT_ID`) to your `.env` (see `.env.sample`). + +## How It Works + +One SDK call creates a session: + +```python +from smallestai import SmallestAI + +client = SmallestAI() +response = client.atoms.web_call.start_web_call_conversation(agent_id="...") +session = response.data +``` + +`start_web_chat_conversation(agent_id=...)` is the same shape for text chat sessions. + +`response.data` contains: + +| Field | What it is | +|-------|------------| +| `token` | Short-lived room access token. Safe to send to the browser. | +| `room_name` | Room UUID pre-created for this session. | +| `host` | WebSocket URL the room client connects to. | +| `conversation_id` | Correlates the session with transcripts and post-call analytics. | +| `call_id` | Call ID surfaced in call logs and analytics endpoints. | + +The intended split: + +- **Server** (this script): holds `SMALLEST_API_KEY`, creates the session, returns `token` + `host` to the frontend. +- **Browser**: passes `token` and `host` to the room client (e.g. the Atoms web SDK or a LiveKit client) to join the live session. It never sees the API key. +- **Afterwards**: `conversation_id` is what `client.atoms.calls.get(id=...)` tracks, so store it if you want transcripts, recordings, or analytics for the session. + +## API Reference + +- [Atoms API Reference](https://docs.smallest.ai/atoms/api-reference) + +## Next Steps + +- [Atoms SDK Web Agent](../atoms_sdk_web_agent/): full browser client that consumes a session like this +- [Analytics](../analytics/): pull post-call data for the conversations you start here diff --git a/voice-agents/web_call_session/requirements.txt b/voice-agents/web_call_session/requirements.txt new file mode 100644 index 0000000..f8cb917 --- /dev/null +++ b/voice-agents/web_call_session/requirements.txt @@ -0,0 +1,2 @@ +# Web call session API requires a recent SDK +smallestai>=5.12.0 diff --git a/voice-agents/web_call_session/start_session.py b/voice-agents/web_call_session/start_session.py new file mode 100644 index 0000000..2af8790 --- /dev/null +++ b/voice-agents/web_call_session/start_session.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Web Call Session + +Starts a browser voice (or text chat) session for an Atoms agent from the +server side. The API key stays on the server; the browser only receives the +short-lived token, host, and room name it needs to join the session. + +Usage: + python start_session.py --agent-id + python start_session.py --agent-id --chat + AGENT_ID=... python start_session.py +""" + +import argparse +import os +import sys + +from dotenv import load_dotenv +from smallestai import SmallestAI + +load_dotenv() + + +def main(): + parser = argparse.ArgumentParser(description="Start a web call or web chat session for an agent") + parser.add_argument( + "--agent-id", + default=os.getenv("AGENT_ID"), + help="Atoms agent ID (defaults to AGENT_ID env var)", + ) + parser.add_argument( + "--chat", + action="store_true", + help="Start a text chat session instead of a voice call", + ) + args = parser.parse_args() + + if not args.agent_id: + print("Error: pass --agent-id or set AGENT_ID", file=sys.stderr) + sys.exit(1) + + client = SmallestAI() # reads SMALLEST_API_KEY from the environment + + if args.chat: + print("Starting web chat session...") + response = client.atoms.web_call.start_web_chat_conversation(agent_id=args.agent_id) + else: + print("Starting web call session...") + response = client.atoms.web_call.start_web_call_conversation(agent_id=args.agent_id) + + session = response.data + + # token + host go to the browser room client; the rest stays server-side. + print(f" token: {session.token}") + print(f" room_name: {session.room_name}") + print(f" host: {session.host}") + print(f" conversation_id: {session.conversation_id}") + print(f" call_id: {session.call_id}") + + print("\nAfter the call, fetch details with:") + print(f' client.atoms.calls.get(id="{session.conversation_id}")') + + +if __name__ == "__main__": + main()