feat(provider): add LiteLLM as embedded AI gateway provider

README.md

@@ -75,7 +75,7 @@ toad acp "fast-agent-acp -x --model sonnet"
 
 The simple declarative syntax lets you concentrate on composing your Prompts and MCP Servers to [build effective agents](https://www.anthropic.com/research/building-effective-agents).
 
-Model support is comprehensive with native support for Anthropic, OpenAI and Google providers as well as Azure, Ollama, Deepseek and dozens of others via TensorZero. Structured Outputs, PDF and Vision support is simple to use and well tested. Passthrough and Playback LLMs enable rapid development and test of Python glue-code for your applications.
+Model support is comprehensive with native support for Anthropic, OpenAI and Google providers as well as Azure, Ollama, Deepseek and dozens of others via TensorZero. The optional `[litellm]` extra adds an embedded [LiteLLM](https://docs.litellm.ai/) gateway provider that routes to 100+ underlying providers (Bedrock, Vertex AI, Cohere, Mistral, Together, Groq, Perplexity, Fireworks, Cerebras, Databricks, …) via standard `*_API_KEY` env vars or a LiteLLM proxy server. Structured Outputs, PDF and Vision support is simple to use and well tested. Passthrough and Playback LLMs enable rapid development and test of Python glue-code for your applications.
 
 Recent features include:
 

pyproject.toml

@@ -66,6 +66,10 @@ bedrock = [
 tensorzero = [
     "tensorzero>=2025.7.5"
 ]
+# For the LiteLLM provider (routes through 100+ underlying providers via the SDK)
+litellm = [
+    "litellm>=1.60,<1.85",
+]
 textual = [
     "textual>=6.2.1",
 ]
@@ -90,7 +94,8 @@ privacy-gpu = [
 all-providers = [
     "azure-identity>=1.14.0",
     "boto3>=1.35.0",
-    "tensorzero>=2025.7.5"
+    "tensorzero>=2025.7.5",
+    "litellm>=1.60,<1.85",
 ]
 
 [build-system]

src/fast_agent/config.py

@@ -1314,6 +1314,53 @@ def _reject_bool_sample_rate(cls, value: Any) -> Any:
         return _reject_bool_number_field(value, field_name="sample_rate")
 
 
+class LiteLLMSettings(BaseModel):
+    """Settings for the LiteLLM provider (embedded SDK or proxy mode)."""
+
+    api_key: str | None = Field(
+        default=None,
+        description=(
+            "Optional LiteLLM proxy API key. Leave unset to let LiteLLM resolve "
+            "credentials from per-provider env vars (ANTHROPIC_API_KEY, "
+            "OPENAI_API_KEY, etc.) at call time."
+        ),
+    )
+    api_base: str | None = Field(
+        default=None,
+        description=(
+            "Optional LiteLLM proxy base URL (e.g. http://localhost:4000). "
+            "When set, every call routes through the proxy."
+        ),
+    )
+    default_model: str | None = Field(
+        default=None,
+        description=(
+            "Default LiteLLM model spec when the LiteLLM provider is selected "
+            "without an explicit model (e.g. 'anthropic/claude-sonnet-4-5')."
+        ),
+    )
+    drop_params: bool = Field(
+        default=True,
+        description=(
+            "Forward `drop_params=True` to litellm.acompletion so unsupported "
+            "kwargs are stripped per backing provider rather than raising."
+        ),
+    )
+    extra_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs forwarded verbatim to litellm.acompletion. Useful "
+            "for routing-specific options like `metadata`, `tags`, `caching`."
+        ),
+    )
+    default_headers: dict[str, str] | None = Field(
+        default=None,
+        description="Custom headers forwarded as `extra_headers` to LiteLLM.",
+    )
+
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
+
+
 class TensorZeroSettings(BaseModel):
     """Settings for using TensorZero LLM gateway."""
 
@@ -1893,6 +1940,9 @@ class Settings(BaseSettings):
     tensorzero: TensorZeroSettings | None = None
     """Settings for using TensorZero inference gateway"""
 
+    litellm: LiteLLMSettings | None = None
+    """Settings for the LiteLLM provider (routes via the LiteLLM SDK)"""
+
     azure: AzureSettings | None = None
     """Settings for using Azure OpenAI Service in the fast-agent application"""
 

src/fast_agent/llm/model_factory.py

@@ -129,6 +129,10 @@ def __call__(self, **kwargs: Any) -> FastAgentLLMProtocol: ...
         "fast_agent.llm.provider.openai.openresponses",
         "OpenResponsesLLM",
     ),
+    Provider.LITELLM: (
+        "fast_agent.llm.provider.openai.llm_litellm",
+        "LiteLLMLLM",
+    ),
 }
 _MODEL_SPECIFIC_CLASS_PATHS: dict[str, tuple[str, str]] = {
     "playback": ("fast_agent.llm.internal.playback", "PlaybackLLM"),

src/fast_agent/llm/model_selection.py

@@ -282,6 +282,116 @@ class ModelSelectionCatalog:
                 model="groq.deepseek-r1-distill-llama-70b",
             ),
         ),
+        # LiteLLM curated set spans the major backing providers so the picker
+        # shows a useful default list when LiteLLM is focused. Use `c` to flip
+        # to the all-catalog scope (~2k models pulled from the LiteLLM SDK).
+        Provider.LITELLM: (
+            CatalogModelEntry(
+                alias="gpt-4o",
+                display_label="OpenAI GPT-4o",
+                model="litellm.openai/gpt-4o",
+            ),
+            CatalogModelEntry(
+                alias="gpt-4o-mini",
+                display_label="OpenAI GPT-4o mini",
+                model="litellm.openai/gpt-4o-mini",
+                fast=True,
+            ),
+            CatalogModelEntry(
+                alias="claude-sonnet",
+                display_label="Anthropic Claude Sonnet",
+                model="litellm.anthropic/claude-sonnet-4-6",
+            ),
+            CatalogModelEntry(
+                alias="claude-haiku",
+                display_label="Anthropic Claude Haiku",
+                model="litellm.anthropic/claude-haiku-4-5",
+                fast=True,
+            ),
+            CatalogModelEntry(
+                alias="claude-opus",
+                display_label="Anthropic Claude Opus",
+                model="litellm.anthropic/claude-opus-4-7",
+            ),
+            CatalogModelEntry(
+                alias="gemini-2.5-pro",
+                display_label="Google Gemini 2.5 Pro",
+                model="litellm.gemini/gemini-2.5-pro",
+            ),
+            CatalogModelEntry(
+                alias="gemini-2.5-flash",
+                display_label="Google Gemini 2.5 Flash",
+                model="litellm.gemini/gemini-2.5-flash",
+                fast=True,
+            ),
+            CatalogModelEntry(
+                alias="vertex-sonnet",
+                display_label="Vertex AI Claude Sonnet",
+                model="litellm.vertex_ai/claude-sonnet-4-5",
+            ),
+            CatalogModelEntry(
+                alias="bedrock-sonnet",
+                display_label="Bedrock Claude Sonnet",
+                model="litellm.bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
+            ),
+            CatalogModelEntry(
+                alias="azure-gpt-4o",
+                display_label="Azure GPT-4o",
+                model="litellm.azure/gpt-4o",
+            ),
+            CatalogModelEntry(
+                alias="cohere-command-r",
+                display_label="Cohere Command R+",
+                model="litellm.cohere/command-r-plus-08-2024",
+            ),
+            CatalogModelEntry(
+                alias="mistral-large",
+                display_label="Mistral Large",
+                model="litellm.mistral/mistral-large-latest",
+            ),
+            CatalogModelEntry(
+                alias="together-llama",
+                display_label="Together Llama 3.3 70B",
+                model="litellm.together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo",
+            ),
+            CatalogModelEntry(
+                alias="groq-llama",
+                display_label="Groq Llama 3.3 70B",
+                model="litellm.groq/llama-3.3-70b-versatile",
+                fast=True,
+            ),
+            CatalogModelEntry(
+                alias="cerebras-llama",
+                display_label="Cerebras Llama 3.3 70B",
+                model="litellm.cerebras/llama-3.3-70b",
+                fast=True,
+            ),
+            CatalogModelEntry(
+                alias="deepseek-chat",
+                display_label="DeepSeek Chat",
+                model="litellm.deepseek/deepseek-chat",
+            ),
+            CatalogModelEntry(
+                alias="xai-grok",
+                display_label="xAI Grok 4",
+                model="litellm.xai/grok-4",
+            ),
+            CatalogModelEntry(
+                alias="perplexity-sonar",
+                display_label="Perplexity Sonar",
+                model="litellm.perplexity/sonar",
+            ),
+            CatalogModelEntry(
+                alias="fireworks-llama",
+                display_label="Fireworks Llama 3.3 70B",
+                model="litellm.fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
+            ),
+            CatalogModelEntry(
+                alias="databricks-claude",
+                display_label="Databricks Claude Sonnet",
+                model="litellm.databricks/databricks-claude-3-7-sonnet",
+            ),
+        ),
         Provider.FAST_AGENT: (
             CatalogModelEntry(
                 alias="passthrough",