diff --git a/README.md b/README.md index 5dcc6c337..ba4357140 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ toad acp "fast-agent-acp -x --model sonnet" The simple declarative syntax lets you concentrate on composing your Prompts and MCP Servers to [build effective agents](https://www.anthropic.com/research/building-effective-agents). -Model support is comprehensive with native support for Anthropic, OpenAI and Google providers as well as Azure, Ollama, Deepseek and dozens of others via TensorZero. Structured Outputs, PDF and Vision support is simple to use and well tested. Passthrough and Playback LLMs enable rapid development and test of Python glue-code for your applications. +Model support is comprehensive with native support for Anthropic, OpenAI and Google providers as well as Azure, Ollama, Deepseek and dozens of others via TensorZero. The optional `[litellm]` extra adds an embedded [LiteLLM](https://docs.litellm.ai/) gateway provider that routes to 100+ underlying providers (Bedrock, Vertex AI, Cohere, Mistral, Together, Groq, Perplexity, Fireworks, Cerebras, Databricks, …) via standard `*_API_KEY` env vars or a LiteLLM proxy server. Structured Outputs, PDF and Vision support is simple to use and well tested. Passthrough and Playback LLMs enable rapid development and test of Python glue-code for your applications. Recent features include: diff --git a/pyproject.toml b/pyproject.toml index d5b74f483..44b98ee10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,10 @@ bedrock = [ tensorzero = [ "tensorzero>=2025.7.5" ] +# For the LiteLLM provider (routes through 100+ underlying providers via the SDK) +litellm = [ + "litellm>=1.60,<1.85", +] textual = [ "textual>=6.2.1", ] @@ -90,7 +94,8 @@ privacy-gpu = [ all-providers = [ "azure-identity>=1.14.0", "boto3>=1.35.0", - "tensorzero>=2025.7.5" + "tensorzero>=2025.7.5", + "litellm>=1.60,<1.85", ] [build-system] diff --git a/src/fast_agent/config.py b/src/fast_agent/config.py index 4d4ddba4b..37dd1349f 100644 --- a/src/fast_agent/config.py +++ b/src/fast_agent/config.py @@ -1314,6 +1314,53 @@ def _reject_bool_sample_rate(cls, value: Any) -> Any: return _reject_bool_number_field(value, field_name="sample_rate") +class LiteLLMSettings(BaseModel): + """Settings for the LiteLLM provider (embedded SDK or proxy mode).""" + + api_key: str | None = Field( + default=None, + description=( + "Optional LiteLLM proxy API key. Leave unset to let LiteLLM resolve " + "credentials from per-provider env vars (ANTHROPIC_API_KEY, " + "OPENAI_API_KEY, etc.) at call time." + ), + ) + api_base: str | None = Field( + default=None, + description=( + "Optional LiteLLM proxy base URL (e.g. http://localhost:4000). " + "When set, every call routes through the proxy." + ), + ) + default_model: str | None = Field( + default=None, + description=( + "Default LiteLLM model spec when the LiteLLM provider is selected " + "without an explicit model (e.g. 'anthropic/claude-sonnet-4-5')." + ), + ) + drop_params: bool = Field( + default=True, + description=( + "Forward `drop_params=True` to litellm.acompletion so unsupported " + "kwargs are stripped per backing provider rather than raising." + ), + ) + extra_kwargs: dict[str, Any] | None = Field( + default=None, + description=( + "Additional kwargs forwarded verbatim to litellm.acompletion. Useful " + "for routing-specific options like `metadata`, `tags`, `caching`." + ), + ) + default_headers: dict[str, str] | None = Field( + default=None, + description="Custom headers forwarded as `extra_headers` to LiteLLM.", + ) + + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) + + class TensorZeroSettings(BaseModel): """Settings for using TensorZero LLM gateway.""" @@ -1893,6 +1940,9 @@ class Settings(BaseSettings): tensorzero: TensorZeroSettings | None = None """Settings for using TensorZero inference gateway""" + litellm: LiteLLMSettings | None = None + """Settings for the LiteLLM provider (routes via the LiteLLM SDK)""" + azure: AzureSettings | None = None """Settings for using Azure OpenAI Service in the fast-agent application""" diff --git a/src/fast_agent/llm/model_factory.py b/src/fast_agent/llm/model_factory.py index 29743e062..056c64ce1 100644 --- a/src/fast_agent/llm/model_factory.py +++ b/src/fast_agent/llm/model_factory.py @@ -129,6 +129,10 @@ def __call__(self, **kwargs: Any) -> FastAgentLLMProtocol: ... "fast_agent.llm.provider.openai.openresponses", "OpenResponsesLLM", ), + Provider.LITELLM: ( + "fast_agent.llm.provider.openai.llm_litellm", + "LiteLLMLLM", + ), } _MODEL_SPECIFIC_CLASS_PATHS: dict[str, tuple[str, str]] = { "playback": ("fast_agent.llm.internal.playback", "PlaybackLLM"), diff --git a/src/fast_agent/llm/model_selection.py b/src/fast_agent/llm/model_selection.py index d59257c02..85cb72b2a 100644 --- a/src/fast_agent/llm/model_selection.py +++ b/src/fast_agent/llm/model_selection.py @@ -282,6 +282,116 @@ class ModelSelectionCatalog: model="groq.deepseek-r1-distill-llama-70b", ), ), + # LiteLLM curated set spans the major backing providers so the picker + # shows a useful default list when LiteLLM is focused. Use `c` to flip + # to the all-catalog scope (~2k models pulled from the LiteLLM SDK). + Provider.LITELLM: ( + CatalogModelEntry( + alias="gpt-4o", + display_label="OpenAI GPT-4o", + model="litellm.openai/gpt-4o", + ), + CatalogModelEntry( + alias="gpt-4o-mini", + display_label="OpenAI GPT-4o mini", + model="litellm.openai/gpt-4o-mini", + fast=True, + ), + CatalogModelEntry( + alias="claude-sonnet", + display_label="Anthropic Claude Sonnet", + model="litellm.anthropic/claude-sonnet-4-6", + ), + CatalogModelEntry( + alias="claude-haiku", + display_label="Anthropic Claude Haiku", + model="litellm.anthropic/claude-haiku-4-5", + fast=True, + ), + CatalogModelEntry( + alias="claude-opus", + display_label="Anthropic Claude Opus", + model="litellm.anthropic/claude-opus-4-7", + ), + CatalogModelEntry( + alias="gemini-2.5-pro", + display_label="Google Gemini 2.5 Pro", + model="litellm.gemini/gemini-2.5-pro", + ), + CatalogModelEntry( + alias="gemini-2.5-flash", + display_label="Google Gemini 2.5 Flash", + model="litellm.gemini/gemini-2.5-flash", + fast=True, + ), + CatalogModelEntry( + alias="vertex-sonnet", + display_label="Vertex AI Claude Sonnet", + model="litellm.vertex_ai/claude-sonnet-4-5", + ), + CatalogModelEntry( + alias="bedrock-sonnet", + display_label="Bedrock Claude Sonnet", + model="litellm.bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + ), + CatalogModelEntry( + alias="azure-gpt-4o", + display_label="Azure GPT-4o", + model="litellm.azure/gpt-4o", + ), + CatalogModelEntry( + alias="cohere-command-r", + display_label="Cohere Command R+", + model="litellm.cohere/command-r-plus-08-2024", + ), + CatalogModelEntry( + alias="mistral-large", + display_label="Mistral Large", + model="litellm.mistral/mistral-large-latest", + ), + CatalogModelEntry( + alias="together-llama", + display_label="Together Llama 3.3 70B", + model="litellm.together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo", + ), + CatalogModelEntry( + alias="groq-llama", + display_label="Groq Llama 3.3 70B", + model="litellm.groq/llama-3.3-70b-versatile", + fast=True, + ), + CatalogModelEntry( + alias="cerebras-llama", + display_label="Cerebras Llama 3.3 70B", + model="litellm.cerebras/llama-3.3-70b", + fast=True, + ), + CatalogModelEntry( + alias="deepseek-chat", + display_label="DeepSeek Chat", + model="litellm.deepseek/deepseek-chat", + ), + CatalogModelEntry( + alias="xai-grok", + display_label="xAI Grok 4", + model="litellm.xai/grok-4", + ), + CatalogModelEntry( + alias="perplexity-sonar", + display_label="Perplexity Sonar", + model="litellm.perplexity/sonar", + ), + CatalogModelEntry( + alias="fireworks-llama", + display_label="Fireworks Llama 3.3 70B", + model="litellm.fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", + ), + CatalogModelEntry( + alias="databricks-claude", + display_label="Databricks Claude Sonnet", + model="litellm.databricks/databricks-claude-3-7-sonnet", + ), + ), Provider.FAST_AGENT: ( CatalogModelEntry( alias="passthrough", diff --git a/src/fast_agent/llm/provider/litellm/__init__.py b/src/fast_agent/llm/provider/litellm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/fast_agent/llm/provider/litellm/llm_litellm.py b/src/fast_agent/llm/provider/litellm/llm_litellm.py new file mode 100644 index 000000000..c09f7f1a7 --- /dev/null +++ b/src/fast_agent/llm/provider/litellm/llm_litellm.py @@ -0,0 +1,217 @@ +"""LiteLLM provider — routes through the LiteLLM SDK to 100+ underlying providers.""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any + +from fast_agent.llm.provider.openai.llm_openai import OpenAILLM +from fast_agent.llm.provider_key_manager import ProviderKeyManager +from fast_agent.llm.provider_types import Provider + +if TYPE_CHECKING: + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionToolParam, + ) + + from fast_agent.types import RequestParams + +# Maps fast-agent config sections to the env vars LiteLLM reads when calling +# the corresponding backing provider. Letting users put credentials in +# `fastagent.config.yaml` (e.g. `anthropic: { api_key: ... }`) and have them +# transparently picked up by LiteLLM avoids re-declaring keys in two places. +# Only well-known backings are bridged; less common LiteLLM providers +# (cohere, mistral, perplexity, ...) still rely on the user exporting the +# matching `*_API_KEY` env var directly, matching standard LiteLLM convention. +_CONFIG_TO_LITELLM_ENV: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = ( + ("anthropic", (("api_key", "ANTHROPIC_API_KEY"), ("base_url", "ANTHROPIC_BASE_URL"))), + ("openai", (("api_key", "OPENAI_API_KEY"), ("base_url", "OPENAI_BASE_URL"))), + ("google", (("api_key", "GEMINI_API_KEY"),)), + ("xai", (("api_key", "XAI_API_KEY"),)), + ("groq", (("api_key", "GROQ_API_KEY"),)), + ("deepseek", (("api_key", "DEEPSEEK_API_KEY"),)), + ("openrouter", (("api_key", "OPENROUTER_API_KEY"),)), +) + + +def _bridge_fastagent_config_to_litellm_env(config: Any) -> None: + """Export config-stored backing creds as env vars LiteLLM understands. + + Only sets each env var when it's not already present, so a user who + explicitly exports `ANTHROPIC_API_KEY` always wins over the config file. + """ + if config is None: + return + for section_name, mappings in _CONFIG_TO_LITELLM_ENV: + section = getattr(config, section_name, None) + if section is None: + continue + for attr, env_key in mappings: + value = getattr(section, attr, None) + if not value or not isinstance(value, str): + continue + if os.getenv(env_key): + continue + os.environ[env_key] = value + + +class _LiteLLMCompletions: + """Mimics the `client.chat.completions` namespace by dispatching to litellm.acompletion.""" + + def __init__(self, parent: "_LiteLLMClientShim") -> None: + self._parent = parent + + async def create(self, **kwargs: Any) -> Any: + import litellm + + merged: dict[str, Any] = dict(kwargs) + if self._parent.api_key and "api_key" not in merged: + merged["api_key"] = self._parent.api_key + if self._parent.base_url and "base_url" not in merged: + # LiteLLM expects "api_base" for proxy routing + merged["api_base"] = self._parent.base_url + if self._parent.default_headers and "extra_headers" not in merged: + merged["extra_headers"] = self._parent.default_headers + if self._parent.timeout is not None and "timeout" not in merged: + merged["timeout"] = self._parent.timeout + merged.setdefault("drop_params", self._parent.drop_params) + + return await litellm.acompletion(**merged) + + +class _LiteLLMChat: + def __init__(self, parent: "_LiteLLMClientShim") -> None: + self.completions = _LiteLLMCompletions(parent) + + +class _LiteLLMFiles: + """Stub for `client.files.*` — LiteLLM routes do not own file uploads.""" + + async def create(self, *args: Any, **kwargs: Any) -> Any: # noqa: ARG002 + raise NotImplementedError( + "File uploads via the LiteLLM provider are not supported. " + "Inline content (base64 / URL) is preferred for multimodal inputs." + ) + + +class _LiteLLMClientShim: + """`AsyncOpenAI`-shaped facade over `litellm.acompletion`. + + Reused by `LiteLLMLLM` so the entire OpenAI streaming, tool-call, + structured-output, and reasoning pipeline in `OpenAILLM` works against + LiteLLM responses, which are normalized to OpenAI shape by design. + """ + + def __init__( + self, + *, + api_key: str | None, + base_url: str | None, + default_headers: dict[str, str] | None, + drop_params: bool, + timeout: float | int | None, + ) -> None: + self.api_key = api_key or None + self.base_url = base_url or None + self.default_headers = default_headers or None + self.drop_params = drop_params + self.timeout = timeout + self.chat = _LiteLLMChat(self) + self.files = _LiteLLMFiles() + + async def __aenter__(self) -> "_LiteLLMClientShim": + return self + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: # noqa: ARG002 + return None + + +DEFAULT_LITELLM_MODEL = "openai/gpt-4o-mini" + + +class LiteLLMLLM(OpenAILLM): + """Native LiteLLM SDK provider. + + Inherits the OpenAI streaming/tool-call/structured-output stack and only + swaps the underlying client to call `litellm.acompletion` — which returns + OpenAI-shape responses for every backing provider. + + Supports two modes: + + - **Embedded SDK (default)**: model spec like `litellm.anthropic/claude-sonnet-4-5` + resolves backing-provider credentials from env vars (`ANTHROPIC_API_KEY`, + `OPENAI_API_KEY`, etc.) per LiteLLM's own resolution rules. + - **Proxy**: set `litellm.api_base` and optionally `litellm.api_key` in + config (or `LITELLM_API_KEY` env var) to route every call through a + LiteLLM proxy server. + """ + + def __init__(self, **kwargs: Any) -> None: + kwargs.pop("provider", None) + super().__init__(provider=Provider.LITELLM, **kwargs) + + cfg = None + if self.context and self.context.config: + cfg = getattr(self.context.config, "litellm", None) + # Bridge backing-provider creds from fastagent.config.yaml into env + # vars so LiteLLM's per-provider auth resolution picks them up. + _bridge_fastagent_config_to_litellm_env(self.context.config) + + self._litellm_api_base: str | None = getattr(cfg, "api_base", None) if cfg else None + self._litellm_drop_params: bool = bool(getattr(cfg, "drop_params", True)) if cfg else True + self._litellm_extra_kwargs: dict[str, Any] = ( + dict(getattr(cfg, "extra_kwargs", {}) or {}) if cfg else {} + ) + + def _initialize_default_params(self, kwargs: dict) -> Any: + return self._initialize_default_params_with_model_fallback(kwargs, DEFAULT_LITELLM_MODEL) + + def _api_key(self) -> str: + try: + return ProviderKeyManager.get_api_key("litellm", self.context.config) + except Exception: + return "" + + def _base_url(self) -> str | None: + return self._litellm_api_base + + # The shim duck-types AsyncOpenAI's `chat.completions.create` surface; we + # accept the Liskov violation because the OpenAILLM call sites only use that + # one method, not the full AsyncOpenAI API. + def _openai_client(self) -> _LiteLLMClientShim: # ty: ignore[invalid-method-override] + timeout: float | int | None = None + params = self.default_request_params + if params is not None: + streaming_timeout = getattr(params, "streaming_timeout", None) + if streaming_timeout is not None: + timeout = streaming_timeout + + return _LiteLLMClientShim( + api_key=self._api_key() or None, + base_url=self._base_url(), + default_headers=self._provider_default_headers(), + drop_params=self._litellm_drop_params, + timeout=timeout, + ) + + async def _normalize_chat_completion_files( + self, + client: Any, + messages: list[ChatCompletionMessageParam], + ) -> list[ChatCompletionMessageParam]: + # OpenAI's file-search file-upload path is not portable through LiteLLM. + # Skip file normalization and pass messages through unchanged. + return messages + + def _prepare_api_request( + self, + messages: list[ChatCompletionMessageParam], + tools: list[ChatCompletionToolParam] | None, + request_params: RequestParams, + ) -> dict[str, Any]: + arguments: dict[str, Any] = super()._prepare_api_request(messages, tools, request_params) + if self._litellm_extra_kwargs: + for key, value in self._litellm_extra_kwargs.items(): + arguments.setdefault(key, value) + return arguments diff --git a/src/fast_agent/llm/provider/openai/llm_openai.py b/src/fast_agent/llm/provider/openai/llm_openai.py index 9d229fd38..a9d4ba901 100644 --- a/src/fast_agent/llm/provider/openai/llm_openai.py +++ b/src/fast_agent/llm/provider/openai/llm_openai.py @@ -741,6 +741,7 @@ async def _process_stream( Provider.GENERIC, Provider.OPENROUTER, Provider.GOOGLE_OAI, + Provider.LITELLM, ] if stream_mode == "manual" or provider_requires_manual: return await self._process_stream_manual(stream, model, capture_filename) diff --git a/src/fast_agent/llm/provider_key_manager.py b/src/fast_agent/llm/provider_key_manager.py index 8f655e6d9..d0f5aa23e 100644 --- a/src/fast_agent/llm/provider_key_manager.py +++ b/src/fast_agent/llm/provider_key_manager.py @@ -31,7 +31,9 @@ "responses": ("openai",), } API_KEY_HINT_TEXT = "" -API_KEYLESS_PROVIDERS: frozenset[str] = frozenset({"anthropic-vertex"}) +# LiteLLM is keyless at the fast-agent layer: the SDK resolves credentials per +# backing provider (e.g. ANTHROPIC_API_KEY, OPENAI_API_KEY) at call time. +API_KEYLESS_PROVIDERS: frozenset[str] = frozenset({"anthropic-vertex", "litellm"}) @runtime_checkable diff --git a/src/fast_agent/llm/provider_model_catalog.py b/src/fast_agent/llm/provider_model_catalog.py index 47ab6b7d6..2d5ec15b6 100644 --- a/src/fast_agent/llm/provider_model_catalog.py +++ b/src/fast_agent/llm/provider_model_catalog.py @@ -61,11 +61,53 @@ def discover(self, config: dict[str, Any]) -> ProviderModelInventory: return ProviderModelInventory() +class LiteLLMModelCatalogAdapter: + """LiteLLM model discovery via the SDK's bundled `models_by_provider` registry. + + Returns every model LiteLLM knows about, prefixed with `litellm.` and the + underlying provider key (e.g. `litellm.anthropic/claude-3-5-sonnet`, + `litellm.bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0`). The same set + is returned for both `current_models` and `all_models` since LiteLLM does + not distinguish a "current" subset. + """ + + provider = Provider.LITELLM + + def discover(self, config: dict[str, Any]) -> ProviderModelInventory: # noqa: ARG002 + try: + import litellm + except ImportError: + return ProviderModelInventory() + + specs: list[str] = [] + seen: set[str] = set() + models_by_provider = getattr(litellm, "models_by_provider", {}) + for backing_provider, models in models_by_provider.items(): + prefix = f"{backing_provider}/" + for model in models: + # Some LiteLLM model strings already include the provider prefix + # (e.g. `gemini/gemini-exp-1206` listed under the `gemini` key). + # Strip it so the spec stays single-prefixed. + model_id = model[len(prefix):] if model.startswith(prefix) else model + spec = f"litellm.{backing_provider}/{model_id}" + if spec in seen: + continue + seen.add(spec) + specs.append(spec) + + if not specs: + return ProviderModelInventory() + + specs_tuple = tuple(specs) + return ProviderModelInventory(current_models=specs_tuple, all_models=specs_tuple) + + class ProviderModelCatalogRegistry: """Registry for provider-specific model discovery adapters.""" _ADAPTERS: ClassVar[dict[Provider, ProviderModelCatalogAdapter]] = { Provider.OPENROUTER: OpenRouterModelCatalogAdapter(), + Provider.LITELLM: LiteLLMModelCatalogAdapter(), } @classmethod diff --git a/src/fast_agent/llm/provider_types.py b/src/fast_agent/llm/provider_types.py index 71ebfcb79..29bc82f29 100644 --- a/src/fast_agent/llm/provider_types.py +++ b/src/fast_agent/llm/provider_types.py @@ -40,3 +40,4 @@ def config_name(self) -> str: CODEX_RESPONSES = ("codexresponses", "Codex Responses") RESPONSES = ("responses", "Responses") OPENRESPONSES = ("openresponses", "OpenResponses") + LITELLM = ("litellm", "LiteLLM") # AI gateway routing to 100+ underlying providers diff --git a/src/fast_agent/ui/model_picker.py b/src/fast_agent/ui/model_picker.py index d9497e459..76463bbc7 100644 --- a/src/fast_agent/ui/model_picker.py +++ b/src/fast_agent/ui/model_picker.py @@ -2,7 +2,7 @@ import shutil from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from prompt_toolkit.application import Application from prompt_toolkit.application.current import get_app_or_none @@ -23,7 +23,6 @@ LLAMACPP_IMPORT_SENTINEL, LLAMACPP_PROVIDER_KEY, REFER_TO_DOCS_PROVIDERS, - ModelAvailability, ModelOption, ModelSource, ProviderActivationAction, @@ -32,12 +31,10 @@ find_provider, model_identity, model_options_for_option, - model_options_for_provider, provider_activation_action, - provider_option_count_label, ) from fast_agent.ui.picker_theme import build_picker_style -from fast_agent.utils.text import strip_to_none +from fast_agent.utils.async_utils import suppress_known_runtime_warnings StyleFragments = list[tuple[str, str]] @@ -53,19 +50,6 @@ class ModelPickerResult: activation_action: ProviderActivationAction | None = None -@dataclass(frozen=True) -class ProviderAvailability: - label: str - style: ModelAvailability - available: bool - - -@dataclass(frozen=True) -class ModelAvailabilityDisplay: - availability: ModelAvailability - marker: str - - @dataclass class PickerState: provider_index: int @@ -73,39 +57,6 @@ class PickerState: source: ModelSource -PROVIDER_DISPLAY_NAME_OVERRIDES = { - "responses": "OpenAI", - "openai": "OpenAI (Legacy)", - "codexresponses": "Codex (Plan)", - "generic": "Generic (ollama)", - "fast-agent": "fast-agent", -} - - -_MODEL_AVAILABILITY_MARKERS: dict[ModelAvailability, str] = { - "active": "✓", - "attention": "!", - "inactive": "✗", -} - - -def _model_availability_display( - model: ModelOption, - *, - provider_available: bool, -) -> ModelAvailabilityDisplay: - if provider_available: - availability: ModelAvailability = "active" - elif model.activation_action is not None: - availability = "attention" - else: - availability = "inactive" - return ModelAvailabilityDisplay( - availability=availability, - marker=_MODEL_AVAILABILITY_MARKERS[availability], - ) - - class _SplitListPicker: LIST_VISIBLE_ROWS = 15 @@ -126,7 +77,7 @@ def __init__( if not self.snapshot.providers: raise ValueError("No providers found in model catalog.") self._initial_provider_name = initial_provider - self._initial_model_spec = strip_to_none(initial_model_spec) + self._initial_model_spec = initial_model_spec.strip() if initial_model_spec else None self.state = PickerState( provider_index=self._initial_provider_index(), @@ -202,8 +153,9 @@ def __init__( def current_provider(self) -> ProviderOption: return self.snapshot.providers[self.state.provider_index] - def _provider_is_available(self, option: ProviderOption) -> bool: - return self._provider_availability(option).available + @staticmethod + def _provider_is_available(option: ProviderOption) -> bool: + return option.active or option.option_key == LLAMACPP_PROVIDER_KEY def _provider_requires_docs_only(self) -> bool: provider = self.current_provider.provider @@ -220,14 +172,8 @@ def _provider_activation_action( @property def current_models(self) -> list[ModelOption]: - provider = self.current_provider.provider - if provider is not None: - return model_options_for_provider( - self.snapshot, - provider, - source=self.state.source, - ) return model_options_for_option( + self.snapshot, self.current_provider, source=self.state.source, ) @@ -296,14 +242,11 @@ def _apply_initial_model_selection(self) -> None: self.current_provider.option_key, ) for source in ("curated", "all"): - if provider_option.provider is None: - models = model_options_for_option(provider_option, source=source) - else: - models = model_options_for_provider( - self.snapshot, - provider_option.provider, - source=source, - ) + models = model_options_for_option( + self.snapshot, + provider_option, + source=source, + ) match_index = _find_initial_model_index(models, self._initial_model_spec) if match_index is None: continue @@ -340,7 +283,7 @@ def _row_style( self, *, selected: bool, - availability: ModelAvailability, + availability: Literal["active", "attention", "inactive"], ) -> str: parts: list[str] = [] if selected: @@ -349,43 +292,96 @@ def _row_style( return " ".join(parts) def _provider_availability_label(self, option: ProviderOption) -> str: - return self._provider_availability(option).label + if option.option_key == LLAMACPP_PROVIDER_KEY: + return "available" + if option.overlay_group and not option.curated_entries: + return "none yet" + if option.active: + return "available" + if option.disabled_reason is not None: + return "disabled" + if self._provider_activation_action(option) is not None: + return "sign in required" + return "not configured" def _provider_availability_style( self, option: ProviderOption, - ) -> ModelAvailability: - return self._provider_availability(option).style - - def _provider_availability(self, option: ProviderOption) -> ProviderAvailability: + ) -> Literal["active", "attention", "inactive"]: if option.option_key == LLAMACPP_PROVIDER_KEY: - return ProviderAvailability("available", "active", True) + return "active" if option.overlay_group and not option.curated_entries: - return ProviderAvailability("none yet", "inactive", False) + return "inactive" if option.active: - return ProviderAvailability("available", "active", True) + return "active" if option.disabled_reason is not None: - return ProviderAvailability("disabled", "attention", False) + return "attention" if self._provider_activation_action(option) is not None: - return ProviderAvailability("sign in required", "attention", False) - return ProviderAvailability("not configured", "inactive", False) + return "attention" + return "inactive" @staticmethod def _provider_display_name(config_name: str, default_name: str) -> str: - return PROVIDER_DISPLAY_NAME_OVERRIDES.get(config_name, default_name) + if config_name == "responses": + return "OpenAI" + if config_name == "openai": + return "OpenAI (Legacy)" + if config_name == "codexresponses": + return "Codex (Plan)" + if config_name == "generic": + return "Local (ollama)" + if config_name == "fast-agent": + return "fast-agent" + + return default_name @classmethod def _provider_display_name_for_option(cls, option: ProviderOption) -> str: if option.display_name is not None: return option.display_name provider = option.provider - if provider is None: - raise ValueError("Provider option requires display_name when provider is unset") + assert provider is not None return cls._provider_display_name( provider.config_name, provider.display_name, ) + @staticmethod + def _provider_entry_count_label(option: ProviderOption) -> str: + if option.option_key == LLAMACPP_PROVIDER_KEY: + return "import flow" + if option.overlay_group: + entry_count = len(option.curated_entries) + suffix = "overlay" if entry_count == 1 else "overlays" + return f"{entry_count} {suffix}" + return f"{len(option.curated_entries)} curated" + + def _overlay_models(self) -> list[ModelOption]: + options: list[ModelOption] = [] + for entry in self.current_provider.curated_entries: + tags: list[str] = [] + if entry.local: + tags.append("local") + if entry.fast: + tags.append("fast") + if not entry.current: + tags.append("legacy") + + suffix = f" ({', '.join(tags)})" if tags else "" + label = f"{(entry.display_label or entry.alias):<19} → {entry.model}{suffix}" + if entry.description: + label = f"{label} — {entry.description}" + options.append( + ModelOption( + spec=entry.model, + label=label, + preset_token=entry.alias, + fast=entry.fast, + curated=entry.current, + ) + ) + return options + def _model_panel_width(self) -> int: cols = self._terminal_cols() return max(42, cols - self._provider_width() - 8) @@ -424,7 +420,7 @@ def _render_provider_panel(self) -> StyleFragments: ) availability = self._provider_availability_label(option) provider_name = self._provider_display_name_for_option(option) - count_label = provider_option_count_label(option) + count_label = self._provider_entry_count_label(option) text = f"{cursor}{provider_name:<16} [{availability}] ({count_label})\n" fragments.append((line_style, text)) return fragments @@ -447,18 +443,29 @@ def _render_model_panel(self) -> StyleFragments: for index, model in enumerate(models): selected = index == self.state.model_index cursor = "❯ " if self._models_focused() and selected else " " - availability_display = _model_availability_display( - model, - provider_available=provider_available, + # Per-model backing-available override (used by LiteLLM rows where + # each model's reachability depends on the backing provider's creds, + # not on the LiteLLM SDK alone). Falls back to provider-level signal. + row_available = ( + provider_available + if model.backing_available is None + else (provider_available and model.backing_available) ) line_style = self._row_style( selected=selected, - availability=availability_display.availability, + availability=( + "active" + if row_available + else "attention" + if model.activation_action is not None + else "inactive" + ), ) + marker = "✓" if row_available else "!" if model.activation_action else "✗" fragments.append( ( line_style, - f"{cursor}{availability_display.marker} " + f"{cursor}{marker} " f"{self._tabulate_model_label(model.label, panel_width=self._model_panel_width())}\n", ) ) @@ -502,180 +509,142 @@ def _create_key_bindings(self) -> KeyBindings: @kb.add("left") def _left(event) -> None: - self._left(event) + self._focus_providers() + event.app.invalidate() @kb.add("right") def _right(event) -> None: - self._right(event) + self._focus_models() + event.app.invalidate() @kb.add("tab") def _tab(event) -> None: - self._tab(event) + event.app.layout.focus_next() + event.app.invalidate() @kb.add("s-tab") def _shift_tab(event) -> None: - self._shift_tab(event) + event.app.layout.focus_previous() + event.app.invalidate() @kb.add("up") def _up(event) -> None: - self._up(event) + if event.app.layout.has_focus(self.provider_window): + self._move_provider(-1) + else: + self._move_model(-1) + event.app.invalidate() @kb.add("down") def _down(event) -> None: - self._down(event) + if event.app.layout.has_focus(self.provider_window): + self._move_provider(1) + else: + self._move_model(1) + event.app.invalidate() @kb.add("c") def _toggle_scope(event) -> None: - self._toggle_scope(event) + self._toggle_source() + event.app.invalidate() @kb.add("enter") def _accept(event) -> None: - self._accept(event) + selected_model = self._selected_model() + if selected_model is None: + return + + provider = self.current_provider + selected_value = ( + selected_model.preset_token + if provider.overlay_group and selected_model.preset_token is not None + else selected_model.spec + ) + if selected_model.activation_action is not None: + event.app.exit( + result=ModelPickerResult( + provider=provider.option_key, + provider_available=self._provider_is_available(provider), + selected_model=selected_value, + resolved_model=None, + source=self.state.source, + refer_to_docs=False, + activation_action=selected_model.activation_action, + ) + ) + return + + if ( + provider.option_key == Provider.GENERIC.config_name + and selected_model.spec == GENERIC_CUSTOM_MODEL_SENTINEL + ): + event.app.exit( + result=ModelPickerResult( + provider=provider.option_key, + provider_available=self._provider_is_available(provider), + selected_model=selected_value, + resolved_model=None, + source=self.state.source, + refer_to_docs=False, + activation_action=None, + ) + ) + return + + if ( + provider.option_key == LLAMACPP_PROVIDER_KEY + and selected_model.spec == LLAMACPP_IMPORT_SENTINEL + ): + event.app.exit( + result=ModelPickerResult( + provider=provider.option_key, + provider_available=self._provider_is_available(provider), + selected_model=selected_model.spec, + resolved_model=None, + source=self.state.source, + refer_to_docs=False, + activation_action=None, + ) + ) + return + + if self._provider_requires_docs_only(): + event.app.exit( + result=ModelPickerResult( + provider=provider.option_key, + provider_available=self._provider_is_available(provider), + selected_model=None, + resolved_model=None, + source=self.state.source, + refer_to_docs=True, + activation_action=None, + ) + ) + return + + event.app.exit( + result=ModelPickerResult( + provider=provider.option_key, + provider_available=self._provider_is_available(provider), + selected_model=selected_value, + resolved_model=selected_value, + source=self.state.source, + refer_to_docs=False, + activation_action=None, + ) + ) @kb.add("q") @kb.add("escape") @kb.add("c-c") def _quit(event) -> None: - self._quit(event) + event.app.exit(result=None) return kb - def _left(self, event) -> None: - self._focus_providers() - event.app.invalidate() - - def _right(self, event) -> None: - self._focus_models() - event.app.invalidate() - - def _tab(self, event) -> None: - event.app.layout.focus_next() - event.app.invalidate() - - def _shift_tab(self, event) -> None: - event.app.layout.focus_previous() - event.app.invalidate() - - def _up(self, event) -> None: - if event.app.layout.has_focus(self.provider_window): - self._move_provider(-1) - else: - self._move_model(-1) - event.app.invalidate() - - def _down(self, event) -> None: - if event.app.layout.has_focus(self.provider_window): - self._move_provider(1) - else: - self._move_model(1) - event.app.invalidate() - - def _toggle_scope(self, event) -> None: - self._toggle_source() - event.app.invalidate() - - def _accept(self, event) -> None: - result = self._selected_result() - if result is not None: - event.app.exit(result=result) - - def _quit(self, event) -> None: - event.app.exit(result=None) - - def _selected_result(self) -> ModelPickerResult | None: - selected_model = self._selected_model() - if selected_model is None: - return None - - provider = self.current_provider - selected_value = self._selected_model_value(provider, selected_model) - - if selected_model.activation_action is not None: - return self._picker_result( - provider, - selected_model=selected_value, - resolved_model=None, - refer_to_docs=False, - activation_action=selected_model.activation_action, - ) - if self._is_generic_custom_model(provider, selected_model): - return self._picker_result( - provider, - selected_model=selected_value, - resolved_model=None, - refer_to_docs=False, - ) - if self._is_llamacpp_import_model(provider, selected_model): - return self._picker_result( - provider, - selected_model=selected_model.spec, - resolved_model=None, - refer_to_docs=False, - ) - if self._provider_requires_docs_only(): - return self._picker_result( - provider, - selected_model=None, - resolved_model=None, - refer_to_docs=True, - ) - return self._picker_result( - provider, - selected_model=selected_value, - resolved_model=selected_value, - refer_to_docs=False, - ) - - def _picker_result( - self, - provider: ProviderOption, - *, - selected_model: str | None, - resolved_model: str | None, - refer_to_docs: bool, - activation_action: ProviderActivationAction | None = None, - ) -> ModelPickerResult: - return ModelPickerResult( - provider=provider.option_key, - provider_available=self._provider_is_available(provider), - selected_model=selected_model, - resolved_model=resolved_model, - source=self.state.source, - refer_to_docs=refer_to_docs, - activation_action=activation_action, - ) - - @staticmethod - def _selected_model_value( - provider: ProviderOption, - selected_model: ModelOption, - ) -> str: - if provider.overlay_group and selected_model.preset_token is not None: - return selected_model.preset_token - return selected_model.spec - - @staticmethod - def _is_generic_custom_model( - provider: ProviderOption, - selected_model: ModelOption, - ) -> bool: - return ( - provider.option_key == Provider.GENERIC.config_name - and selected_model.spec == GENERIC_CUSTOM_MODEL_SENTINEL - ) - - @staticmethod - def _is_llamacpp_import_model( - provider: ProviderOption, - selected_model: ModelOption, - ) -> bool: - return ( - provider.option_key == LLAMACPP_PROVIDER_KEY - and selected_model.spec == LLAMACPP_IMPORT_SENTINEL - ) - def run(self) -> ModelPickerResult | None: - result = self.app.run() + with suppress_known_runtime_warnings(): + result = self.app.run() if result is None: return None if isinstance(result, ModelPickerResult): @@ -683,7 +652,8 @@ def run(self) -> ModelPickerResult | None: return None async def run_async(self) -> ModelPickerResult | None: - result = await self.app.run_async() + with suppress_known_runtime_warnings(): + result = await self.app.run_async() if result is None: return None if isinstance(result, ModelPickerResult): @@ -734,7 +704,7 @@ def _find_initial_model_index( return None for index, option in enumerate(options): - if normalized_spec in (option.spec, option.preset_token): + if option.spec == normalized_spec or option.preset_token == normalized_spec: return index target_identity = model_identity(normalized_spec) diff --git a/src/fast_agent/ui/model_picker_common.py b/src/fast_agent/ui/model_picker_common.py index 31dc6591d..57a86302a 100644 --- a/src/fast_agent/ui/model_picker_common.py +++ b/src/fast_agent/ui/model_picker_common.py @@ -1,59 +1,51 @@ from __future__ import annotations import os -from collections.abc import Callable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, TypeGuard +from typing import TYPE_CHECKING, Any, Literal from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from fast_agent.config import get_settings -from fast_agent.constants import DEFAULT_ENVIRONMENT_DIR, FAST_AGENT_RUNTIME_ENVIRONMENT +from fast_agent.constants import DEFAULT_ENVIRONMENT_DIR from fast_agent.llm.model_database import ModelDatabase +from fast_agent.llm.model_factory import ModelFactory from fast_agent.llm.model_overlays import load_model_overlay_registry from fast_agent.llm.model_selection import CatalogModelEntry, ModelSelectionCatalog from fast_agent.llm.provider.anthropic.vertex_config import ( + anthropic_vertex_intent, anthropic_vertex_ready, ) from fast_agent.llm.provider_key_manager import ProviderKeyManager +from fast_agent.llm.provider_model_catalog import ProviderModelCatalogRegistry from fast_agent.llm.provider_types import Provider from fast_agent.llm.reasoning_effort import available_reasoning_values, format_reasoning_setting -from fast_agent.utils.action_normalization import on_off_label -from fast_agent.utils.collections import unique_preserve_order -from fast_agent.utils.count_display import format_count -from fast_agent.utils.text import strip_str_to_none if TYPE_CHECKING: from pathlib import Path ModelSource = Literal["curated", "all"] ProviderActivationAction = Literal["codex-login"] -ModelAvailability = Literal["active", "attention", "inactive"] -MODEL_AVAILABILITIES: tuple[ModelAvailability, ...] = ("active", "attention", "inactive") KEEP_VALUE = "__keep__" DEFAULT_VALUE = "__default__" - -def is_model_availability(value: object) -> TypeGuard[ModelAvailability]: - return value in MODEL_AVAILABILITIES - - PICKER_PROVIDER_ORDER: tuple[Provider, ...] = ( + Provider.LITELLM, Provider.RESPONSES, Provider.OPENRESPONSES, Provider.CODEX_RESPONSES, Provider.ANTHROPIC, + Provider.ANTHROPIC_VERTEX, Provider.HUGGINGFACE, + Provider.OPENAI, + Provider.GENERIC, Provider.GOOGLE, Provider.XAI, - Provider.DEEPSEEK, - Provider.GENERIC, - Provider.ANTHROPIC_VERTEX, - Provider.OPENAI, Provider.GROQ, - Provider.AZURE, - Provider.BEDROCK, + Provider.DEEPSEEK, Provider.ALIYUN, Provider.OPENROUTER, + Provider.AZURE, + Provider.BEDROCK, Provider.FAST_AGENT, ) @@ -68,9 +60,6 @@ def is_model_availability(value: object) -> TypeGuard[ModelAvailability]: ANTHROPIC_VERTEX_PROVIDER_KEY = "anthropic-vertex" LLAMACPP_PROVIDER_KEY = "llamacpp" LLAMACPP_IMPORT_SENTINEL = "llamacpp.__import__" -PROVIDER_PREFIX_DELIMITERS = ("/", ".") -ProviderActiveCheck = Callable[[dict[str, Any]], bool] -ModelSpecTransform = Callable[[str], str] @dataclass(frozen=True) @@ -87,16 +76,14 @@ class ProviderOption: def option_key(self) -> str: if self.key is not None: return self.key - if self.provider is None: - raise ValueError("Provider option requires key when provider is unset") + assert self.provider is not None return self.provider.config_name @property def option_display_name(self) -> str: if self.display_name is not None: return self.display_name - if self.provider is None: - raise ValueError("Provider option requires display_name when provider is unset") + assert self.provider is not None return self.provider.display_name @@ -108,9 +95,10 @@ class ModelOption: fast: bool = False curated: bool = False activation_action: ProviderActivationAction | None = None - - -SyntheticProviderOptionFactory = Callable[[Provider], list[ModelOption] | None] + # When set, overrides the parent provider's availability for marker rendering. + # Used by LiteLLM rows where each model's reachability depends on the + # backing provider's credentials, not on the LiteLLM SDK alone. + backing_available: bool | None = None @dataclass(frozen=True) @@ -146,45 +134,122 @@ def _provider_is_active(provider: Provider, config_payload: dict[str, Any]) -> b if ProviderKeyManager.get_env_var(provider.config_name): return True - if active_check := _PROVIDER_ACTIVE_CHECKS.get(provider): - return active_check(config_payload) - - return provider in {Provider.FAST_AGENT, Provider.GENERIC} - + if provider == Provider.GOOGLE: + google_cfg = config_payload.get("google") + if isinstance(google_cfg, dict): + vertex_cfg = google_cfg.get("vertex_ai") + if isinstance(vertex_cfg, dict) and bool(vertex_cfg.get("enabled")): + return True + + if provider == Provider.AZURE: + azure_cfg = config_payload.get("azure") + if isinstance(azure_cfg, dict): + use_default = bool(azure_cfg.get("use_default_azure_credential")) + base_url = azure_cfg.get("base_url") + if use_default and isinstance(base_url, str) and bool(base_url.strip()): + return True + + if provider == Provider.CODEX_RESPONSES: + try: + from fast_agent.llm.provider.openai.codex_oauth import get_codex_token_status + + status = get_codex_token_status() + if bool(status.get("present")): + return True + except Exception: + pass + + if provider in {Provider.FAST_AGENT, Provider.GENERIC}: + return True -def _google_vertex_is_active(config_payload: dict[str, Any]) -> bool: - google_cfg = config_payload.get("google") - if not isinstance(google_cfg, dict): - return False - vertex_cfg = google_cfg.get("vertex_ai") - return isinstance(vertex_cfg, dict) and bool(vertex_cfg.get("enabled")) + # LiteLLM is "available" once the SDK is importable. The SDK resolves + # backing-provider credentials at call time, so showing the row as available + # lets the user pick a model and have LiteLLM raise its own clear error if + # the routed provider's API key is missing. + if provider == Provider.LITELLM: + try: + import litellm # noqa: F401 + return True + except ImportError: + return False -def _azure_default_credential_is_active(config_payload: dict[str, Any]) -> bool: - azure_cfg = config_payload.get("azure") - if not isinstance(azure_cfg, dict): - return False - use_default = bool(azure_cfg.get("use_default_azure_credential")) - base_url = azure_cfg.get("base_url") - normalized_base_url = strip_str_to_none(base_url) - return use_default and normalized_base_url is not None + return False -def _codex_oauth_is_active(_config_payload: dict[str, Any]) -> bool: - try: - from fast_agent.llm.provider.openai.codex_oauth import get_codex_token_status +def litellm_backing_provider_for_spec(spec: str) -> str | None: + """Extract the backing provider from a `litellm./` spec.""" + if not spec.startswith("litellm."): + return None + body = spec[len("litellm.") :] + head = body.split("/", 1)[0] + return head or None + + +_LITELLM_BACKING_ENV_KEYS: dict[str, tuple[str, ...]] = { + "openai": ("OPENAI_API_KEY",), + "anthropic": ("ANTHROPIC_API_KEY",), + "anthropic_text": ("ANTHROPIC_API_KEY",), + "azure": ("AZURE_API_KEY", "AZURE_OPENAI_API_KEY"), + "azure_ai": ("AZURE_AI_API_KEY", "AZURE_API_KEY"), + "azure_anthropic": ("AZURE_API_KEY",), + "bedrock": ("AWS_ACCESS_KEY_ID", "AWS_BEARER_TOKEN_BEDROCK"), + "amazon_nova": ("AWS_ACCESS_KEY_ID",), + "vertex_ai": ("GOOGLE_APPLICATION_CREDENTIALS", "VERTEXAI_PROJECT", "VERTEX_PROJECT"), + "vertex_ai_beta": ("GOOGLE_APPLICATION_CREDENTIALS",), + "gemini": ("GEMINI_API_KEY", "GOOGLE_API_KEY"), + "groq": ("GROQ_API_KEY",), + "cohere": ("COHERE_API_KEY",), + "cohere_chat": ("COHERE_API_KEY",), + "mistral": ("MISTRAL_API_KEY", "CODESTRAL_API_KEY"), + "codestral": ("CODESTRAL_API_KEY",), + "deepseek": ("DEEPSEEK_API_KEY",), + "xai": ("XAI_API_KEY",), + "perplexity": ("PERPLEXITYAI_API_KEY",), + "fireworks_ai": ("FIREWORKS_API_KEY", "FIREWORKS_AI_API_KEY"), + "cerebras": ("CEREBRAS_API_KEY",), + "together_ai": ("TOGETHERAI_API_KEY", "TOGETHER_API_KEY"), + "databricks": ("DATABRICKS_API_KEY",), + "ai21": ("AI21_API_KEY",), + "openrouter": ("OPENROUTER_API_KEY",), + "huggingface": ("HF_TOKEN", "HUGGINGFACE_API_KEY"), + "cloudflare": ("CLOUDFLARE_API_KEY",), + "replicate": ("REPLICATE_API_KEY",), + "anyscale": ("ANYSCALE_API_KEY",), + "watsonx": ("WATSONX_API_KEY", "WATSONX_TOKEN"), + "sambanova": ("SAMBANOVA_API_KEY",), + "nvidia_nim": ("NVIDIA_NIM_API_KEY",), + "deepinfra": ("DEEPINFRA_API_KEY",), + "voyage": ("VOYAGE_API_KEY",), + "novita": ("NOVITA_API_KEY",), + "moonshot": ("MOONSHOT_API_KEY",), + "dashscope": ("DASHSCOPE_API_KEY",), + "baseten": ("BASETEN_API_KEY",), + "lambda_ai": ("LAMBDA_API_KEY",), + "minimax": ("MINIMAX_API_KEY",), + "friendliai": ("FRIENDLIAI_API_KEY",), + "nlp_cloud": ("NLP_CLOUD_API_KEY",), + # Local / OAuth-driven backings (kept out of the map so we don't probe OAuth flows) + # github_copilot, ollama, llamafile -- treated as "unknown" -> falls back to provider-level +} - status = get_codex_token_status() - return bool(status.get("present")) - except Exception: - return False +def litellm_backing_creds_present(spec: str) -> bool | None: + """Return True/False if any expected env key for the spec's backing is set. -_PROVIDER_ACTIVE_CHECKS: dict[Provider, ProviderActiveCheck] = { - Provider.GOOGLE: _google_vertex_is_active, - Provider.AZURE: _azure_default_credential_is_active, - Provider.CODEX_RESPONSES: _codex_oauth_is_active, -} + Uses a static map rather than `litellm.validate_environment(...)` because + that API can trigger OAuth device-code flows for some backings (notably + `github_copilot`), which would block the wizard render. Returns None when + the backing isn't in the map so callers can fall back to the provider's + SDK-level availability without showing a misleading ✗. + """ + backing = litellm_backing_provider_for_spec(spec) + if backing is None: + return None + keys = _LITELLM_BACKING_ENV_KEYS.get(backing) + if not keys: + return None + return any(os.getenv(key) for key in keys) def _catalog_options_from_entries( @@ -192,117 +257,106 @@ def _catalog_options_from_entries( *, provider: Provider, source: ModelSource, - spec_transform: ModelSpecTransform | None = None, - filter_current: bool = True, + spec_transform: Any = None, + discovered_specs: tuple[str, ...] = (), ) -> list[ModelOption]: - transform = spec_transform or _identity_model_spec + transform = spec_transform or (lambda value: value) - entry_options: list[ModelOption] = [] + curated_options: list[ModelOption] = [] for entry in entries: spec = transform(entry.model) - entry_options.append( + tags: list[str] = [] + if entry.local: + tags.append("local") + if entry.fast: + tags.append("fast") + if not entry.current: + tags.append("legacy") + + suffix = f" ({', '.join(tags)})" if tags else "" + entry_label = entry.display_label or entry.alias + label = f"{entry_label:<19} → {spec}{suffix}" + if entry.description: + label = f"{label} — {entry.description}" + backing_available = ( + litellm_backing_creds_present(spec) if provider == Provider.LITELLM else None + ) + curated_options.append( ModelOption( spec=spec, - label=format_catalog_model_entry_label(entry, spec=spec), + label=label, preset_token=entry.alias, fast=entry.fast, curated=entry.current, + backing_available=backing_available, ) ) if source == "curated": - if not filter_current: - return entry_options - return [ - option for entry, option in zip(entries, entry_options, strict=True) if entry.current - ] + return curated_options seen_identities: set[tuple[Provider, str]] = set() - options: list[ModelOption] = list(entry_options) - for option in entry_options: - identity = model_identity(option.spec) + seen_specs: set[str] = set() + options: list[ModelOption] = list(curated_options) + for curated in curated_options: + seen_specs.add(curated.spec) + identity = model_identity(curated.spec) if identity is not None: seen_identities.add(identity) + is_litellm_provider = provider == Provider.LITELLM + for spec in _static_provider_models(provider): transformed_spec = transform(spec) + if transformed_spec in seen_specs: + continue identity = model_identity(transformed_spec) if identity is not None and identity in seen_identities: continue if identity is not None: seen_identities.add(identity) - options.append(ModelOption(spec=transformed_spec, label=f"{transformed_spec} (catalog)")) - - return options - - -def _identity_model_spec(model_spec: str) -> str: - return model_spec - - -def catalog_model_entry_tags(entry: CatalogModelEntry) -> tuple[str, ...]: - tags: list[str] = [] - if entry.local: - tags.append("local") - if entry.fast: - tags.append("fast") - if not entry.current: - tags.append("legacy") - return tuple(tags) - - -def format_catalog_model_entry_label(entry: CatalogModelEntry, *, spec: str | None = None) -> str: - resolved_spec = spec or entry.model - tags = catalog_model_entry_tags(entry) - suffix = f" ({', '.join(tags)})" if tags else "" - label = f"{(entry.display_label or entry.alias):<19} → {resolved_spec}{suffix}" - if entry.description: - label = f"{label} — {entry.description}" - return label - - -def _generic_provider_model_options(_provider: Provider) -> list[ModelOption]: - return [ - ModelOption( - spec=GENERIC_CUSTOM_MODEL_SENTINEL, - label="Enter local model string (e.g. llama3.2)", + seen_specs.add(transformed_spec) + backing_available = ( + litellm_backing_creds_present(transformed_spec) if is_litellm_provider else None ) - ] - - -def _refer_to_docs_provider_model_options(provider: Provider) -> list[ModelOption]: - return [ - ModelOption( - spec=f"{provider.config_name}.refer-to-docs", - label="Refer to docs (provider-specific setup)", + options.append( + ModelOption( + spec=transformed_spec, + label=f"{transformed_spec} (catalog)", + backing_available=backing_available, + ) ) - ] - - -_SYNTHETIC_PROVIDER_OPTION_FACTORIES: dict[Provider, SyntheticProviderOptionFactory] = { - Provider.GENERIC: _generic_provider_model_options, - **{provider: _refer_to_docs_provider_model_options for provider in REFER_TO_DOCS_PROVIDERS}, -} + for spec in discovered_specs: + transformed_spec = transform(spec) + if transformed_spec in seen_specs: + continue + identity = model_identity(transformed_spec) + if identity is not None and identity in seen_identities: + continue + if identity is not None: + seen_identities.add(identity) + seen_specs.add(transformed_spec) + backing_available = ( + litellm_backing_creds_present(transformed_spec) if is_litellm_provider else None + ) + options.append( + ModelOption( + spec=transformed_spec, + label=f"{transformed_spec} (catalog)", + backing_available=backing_available, + ) + ) -def _synthetic_provider_model_options(provider: Provider) -> list[ModelOption] | None: - factory = _SYNTHETIC_PROVIDER_OPTION_FACTORIES.get(provider) - if factory is None: - return None - return factory(provider) + return options def model_options_for_option( + snapshot: ModelPickerSnapshot, option: ProviderOption, *, source: ModelSource, ) -> list[ModelOption]: - provider = option.provider - if provider is not None: - synthetic_options = _synthetic_provider_model_options(provider) - if synthetic_options is not None: - return synthetic_options - if option.option_key == LLAMACPP_PROVIDER_KEY: return [ ModelOption( @@ -316,15 +370,21 @@ def model_options_for_option( option.curated_entries, provider=Provider.ANTHROPIC, source="curated", - filter_current=False, ) - if provider is None: - raise ValueError(f"Provider option '{option.option_key}' has no model provider") + provider = option.provider + assert provider is not None + + discovered_specs: tuple[str, ...] = () + if source == "all": + discovered = ProviderModelCatalogRegistry.discover(provider, snapshot.config_payload) + discovered_specs = discovered.all_models + return _catalog_options_from_entries( option.curated_entries, provider=provider, source=source, + discovered_specs=discovered_specs, ) @@ -361,7 +421,10 @@ def build_snapshot( ) for overlay in overlay_registry.overlays ) - overlay_group_active = bool(overlay_entries) + if overlay_entries: + overlay_group_active = True + else: + overlay_group_active = False providers.append( ProviderOption( provider=None, @@ -373,6 +436,8 @@ def build_snapshot( ) ) for provider in PICKER_PROVIDER_ORDER: + if provider == Provider.ANTHROPIC_VERTEX and not anthropic_vertex_intent(config_payload): + continue entries = tuple( entry for entry in ModelSelectionCatalog.list_entries( @@ -386,22 +451,11 @@ def build_snapshot( ) if not entries and not has_special_picker_flow: continue - if provider == Provider.DEEPSEEK: - providers.append( - ProviderOption( - provider=None, - active=False, - curated_entries=(), - key=LLAMACPP_PROVIDER_KEY, - display_name="llama.cpp", - ) - ) providers.append( ProviderOption( provider=provider, active=provider in active_providers, curated_entries=entries, - display_name=("Generic (ollama)" if provider == Provider.GENERIC else None), disabled_reason=( anthropic_vertex_ready(config_payload)[1] if provider == Provider.ANTHROPIC_VERTEX and provider not in active_providers @@ -409,6 +463,16 @@ def build_snapshot( ), ) ) + if provider == Provider.GENERIC: + providers.append( + ProviderOption( + provider=None, + active=False, + curated_entries=(), + key=LLAMACPP_PROVIDER_KEY, + display_name="llama.cpp", + ) + ) return ModelPickerSnapshot(providers=tuple(providers), config_payload=config_payload) @@ -419,133 +483,58 @@ def _load_overlay_registry_for_snapshot( config_payload: dict[str, Any], start_path: Path | None, ): - env_dir = config_payload.get("environment_dir") - normalized_env_dir = _normalized_overlay_env_dir(env_dir) - candidate_starts = _overlay_candidate_starts( - config_path=config_path, - env_dir=env_dir, - normalized_env_dir=normalized_env_dir, - start_path=start_path, - ) - - if normalized_env_dir is None and (config_path is not None or start_path is not None): - normalized_env_dir = DEFAULT_ENVIRONMENT_DIR - - return _first_overlay_registry_with_entries( - _dedupe_paths(candidate_starts), - env_dir=normalized_env_dir, - ) - - -def _normalized_overlay_env_dir(env_dir: object) -> str | Path | None: from pathlib import Path as _Path - if isinstance(env_dir, (str, _Path)): - return env_dir - return os.getenv(FAST_AGENT_RUNTIME_ENVIRONMENT) or os.getenv("ENVIRONMENT_DIR") - - -def _overlay_candidate_starts( - *, - config_path: str | Path | None, - env_dir: object, - normalized_env_dir: str | Path | None, - start_path: Path | None, -) -> list[Path]: - from pathlib import Path as _Path + env_dir = config_payload.get("environment_dir") + normalized_env_dir = env_dir if isinstance(env_dir, (str, _Path)) else None + candidate_starts: list[_Path] = [] if config_path is not None: - return _config_overlay_candidate_starts( - config_path=config_path, - env_dir=env_dir, - normalized_env_dir=normalized_env_dir, - ) - - if start_path is not None: - return [_Path(start_path).expanduser().resolve()] - return [_Path.cwd().resolve()] - - -def _config_overlay_candidate_starts( - *, - config_path: str | Path, - env_dir: object, - normalized_env_dir: str | Path | None, -) -> list[Path]: - from pathlib import Path as _Path - - config_file = _Path(config_path).expanduser().resolve() - candidate_starts: list[Path] = [config_file.parent] - relative_env_dir = _relative_overlay_env_dir( - env_dir=env_dir, - normalized_env_dir=normalized_env_dir, - ) - project_root = _project_root_for_env_config(config_file.parent, relative_env_dir) - if project_root is not None: - candidate_starts.append(project_root) - return candidate_starts - - -def _relative_overlay_env_dir( - *, - env_dir: object, - normalized_env_dir: str | Path | None, -) -> Path | None: - from pathlib import Path as _Path - - if env_dir is None: - return _Path(DEFAULT_ENVIRONMENT_DIR) - if normalized_env_dir is None: - raise ValueError("environment_dir must be a string or path") - - env_path = _Path(normalized_env_dir).expanduser() - if env_path.is_absolute(): - return None - return env_path - - -def _project_root_for_env_config(config_dir: Path, relative_env_dir: Path | None) -> Path | None: - if relative_env_dir is None or not relative_env_dir.parts: - return None - - env_parts = relative_env_dir.parts - parent_parts = config_dir.parts - if len(env_parts) > len(parent_parts) or parent_parts[-len(env_parts) :] != env_parts: - return None - - project_root = config_dir - for _ in env_parts: - project_root = project_root.parent - if project_root == config_dir: - return None - return project_root - - -def _dedupe_paths(candidate_starts: list[Path]) -> list[Path]: - return unique_preserve_order(candidate_starts) - - -def _first_overlay_registry_with_entries( - ordered_starts: list[Path], - *, - env_dir: str | Path | None, -): - from pathlib import Path as _Path + config_file = _Path(config_path).expanduser().resolve() + candidate_starts.append(config_file.parent) + + relative_env_dir: _Path | None = None + if normalized_env_dir is None: + relative_env_dir = _Path(DEFAULT_ENVIRONMENT_DIR) + else: + env_path = _Path(normalized_env_dir).expanduser() + if not env_path.is_absolute(): + relative_env_dir = env_path + + if relative_env_dir is not None and relative_env_dir.parts: + env_parts = relative_env_dir.parts + parent_parts = config_file.parent.parts + if len(env_parts) <= len(parent_parts) and parent_parts[-len(env_parts) :] == env_parts: + project_root = config_file.parent + for _ in env_parts: + project_root = project_root.parent + if project_root != config_file.parent: + candidate_starts.append(project_root) + + if config_path is None and start_path is not None: + candidate_starts.append(_Path(start_path).expanduser().resolve()) + + if config_path is None and start_path is None: + candidate_starts.append(_Path.cwd().resolve()) + + seen: set[_Path] = set() + ordered_starts: list[_Path] = [] + for candidate in candidate_starts: + if candidate in seen: + continue + seen.add(candidate) + ordered_starts.append(candidate) fallback_registry = None - for overlay_start_path in ordered_starts: - registry = load_model_overlay_registry( - start_path=overlay_start_path, - env_dir=env_dir, - ) + for start_path in ordered_starts: + registry = load_model_overlay_registry(start_path=start_path, env_dir=normalized_env_dir) if fallback_registry is None: fallback_registry = registry if registry.overlays: return registry - if fallback_registry is not None: - return fallback_registry - return load_model_overlay_registry(start_path=_Path.cwd().resolve(), env_dir=env_dir) + assert fallback_registry is not None + return fallback_registry def find_provider(snapshot: ModelPickerSnapshot, provider_name: str) -> ProviderOption: @@ -555,31 +544,22 @@ def find_provider(snapshot: ModelPickerSnapshot, provider_name: str) -> Provider raise ValueError(f"Unknown provider: {provider_name}") -def provider_option_status_label(option: ProviderOption) -> str: - if option.option_key == LLAMACPP_PROVIDER_KEY: - return "active" - if option.active: - return "active" - if option.disabled_reason: - return "disabled" - return "inactive" - - -def provider_option_count_label(option: ProviderOption) -> str: +def build_provider_label(option: ProviderOption) -> str: if option.option_key == LLAMACPP_PROVIDER_KEY: - return "import flow" + status = "active" + count_text = "import flow" + return f"{option.option_display_name:<16} [{status}] · {count_text}" + status = "active" if option.active else "disabled" if option.disabled_reason else "inactive" curated_count = len(option.curated_entries) if option.overlay_group: - return format_count(curated_count, "overlay") - return format_count(curated_count, "curated model") - - -def build_provider_label(option: ProviderOption) -> str: - count_text = provider_option_count_label(option) - return ( - f"{option.option_display_name:<16} [{provider_option_status_label(option)}] · {count_text}" - ) + entry_text = "overlay" if curated_count == 1 else "overlays" + count_text = f"{curated_count} {entry_text}" + else: + count_text = f"{curated_count} curated model" + if curated_count != 1: + count_text += "s" + return f"{option.option_display_name:<16} [{status}] · {count_text}" def active_provider_names(snapshot: ModelPickerSnapshot) -> list[str]: @@ -588,10 +568,15 @@ def active_provider_names(snapshot: ModelPickerSnapshot) -> list[str]: def has_explicit_provider_prefix(model_spec: str) -> bool: provider_names = {provider.config_name for provider in Provider} - for delimiter in PROVIDER_PREFIX_DELIMITERS: - prefix, separator, rest = model_spec.partition(delimiter) - if prefix and separator and rest and prefix in provider_names: - return True + + slash_prefix, _, slash_rest = model_spec.partition("/") + if slash_prefix and slash_rest and slash_prefix in provider_names: + return True + + dot_prefix, _, dot_rest = model_spec.partition(".") + if dot_prefix and dot_rest and dot_prefix in provider_names: + return True + return False @@ -614,8 +599,6 @@ def infer_initial_picker_provider(model_spec: str | None) -> str | None: if not normalized: return None - from fast_agent.llm.model_factory import ModelFactory - try: parsed = ModelFactory.parse_model_string( normalized, @@ -639,8 +622,6 @@ def provider_activation_action( def model_identity(model_spec: str) -> tuple[Provider, str] | None: - from fast_agent.llm.model_factory import ModelFactory - try: parsed = ModelFactory.parse_model_string(model_spec) except Exception: @@ -669,9 +650,21 @@ def model_options_for_provider( *, source: ModelSource, ) -> list[ModelOption]: - synthetic_options = _synthetic_provider_model_options(provider) - if synthetic_options is not None: - return synthetic_options + if provider == Provider.GENERIC: + return [ + ModelOption( + spec=GENERIC_CUSTOM_MODEL_SENTINEL, + label="Enter local model string (e.g. llama3.2)", + ) + ] + + if provider in REFER_TO_DOCS_PROVIDERS: + return [ + ModelOption( + spec=f"{provider.config_name}.refer-to-docs", + label="Refer to docs (provider-specific setup)", + ) + ] provider_option = find_provider(snapshot, provider.config_name) activation_action = provider_activation_action(snapshot, provider) @@ -691,8 +684,6 @@ def model_options_for_provider( def model_capabilities(model_spec: str) -> ModelCapabilities: - from fast_agent.llm.model_factory import ModelFactory - resolved = ModelFactory.resolve_model_spec(model_spec) parsed = resolved.model_config reasoning_spec = resolved.reasoning_effort_spec @@ -711,7 +702,7 @@ def model_capabilities(model_spec: str) -> ModelCapabilities: current_reasoning=format_reasoning_setting(parsed.reasoning_effort), default_reasoning=default_reasoning, web_search_supported=( - parsed.provider in {Provider.RESPONSES, Provider.CODEX_RESPONSES, Provider.XAI} + parsed.provider in {Provider.RESPONSES, Provider.CODEX_RESPONSES} or ( parsed.provider == Provider.ANTHROPIC and resolved.anthropic_web_search_version is not None @@ -759,20 +750,23 @@ def apply_option_overrides( """ result = model_spec - overrides = { - "reasoning": reasoning_value, - "web_search": web_search_value, - "context": context_value, - } - for key, selected_value in overrides.items(): - if selected_value is None: - continue - target = None if selected_value == DEFAULT_VALUE else selected_value - result = _update_query_param(result, key=key, value=target) + + if reasoning_value is not None: + target = None if reasoning_value == DEFAULT_VALUE else reasoning_value + result = _update_query_param(result, key="reasoning", value=target) + + if web_search_value is not None: + target = None if web_search_value == DEFAULT_VALUE else web_search_value + result = _update_query_param(result, key="web_search", value=target) + + if context_value is not None: + target = None if context_value == DEFAULT_VALUE else context_value + result = _update_query_param(result, key="context", value=target) + return result def web_search_display(value: bool | None) -> str: if value is None: return "default" - return on_off_label(value) + return "on" if value else "off" diff --git a/tests/unit/fast_agent/llm/test_litellm_provider.py b/tests/unit/fast_agent/llm/test_litellm_provider.py new file mode 100644 index 000000000..a5df333ff --- /dev/null +++ b/tests/unit/fast_agent/llm/test_litellm_provider.py @@ -0,0 +1,375 @@ +"""Unit tests for the LiteLLM provider integration.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from fast_agent.llm.model_factory import ModelFactory +from fast_agent.llm.provider_key_manager import ProviderKeyManager +from fast_agent.llm.provider_model_catalog import ( + LiteLLMModelCatalogAdapter, + ProviderModelCatalogRegistry, +) +from fast_agent.llm.provider_types import Provider + + +def test_provider_enum_includes_litellm() -> None: + assert Provider.LITELLM.config_name == "litellm" + assert Provider.LITELLM.display_name == "LiteLLM" + + +def test_picker_provider_order_includes_litellm() -> None: + from fast_agent.ui.model_picker_common import PICKER_PROVIDER_ORDER + + assert Provider.LITELLM in PICKER_PROVIDER_ORDER + + +def test_provider_is_active_true_when_litellm_importable() -> None: + from fast_agent.ui.model_picker_common import _provider_is_active + + # litellm is in test deps; this should resolve to True + assert _provider_is_active(Provider.LITELLM, {}) is True + + +def test_provider_is_active_false_when_litellm_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """Wizard should not advertise LiteLLM as available if the SDK isn't installed.""" + import builtins + + from fast_agent.ui.model_picker_common import _provider_is_active + + real_import = builtins.__import__ + + def _no_litellm(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "litellm" or name.startswith("litellm."): + raise ImportError("litellm not installed (simulated)") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_litellm) + assert _provider_is_active(Provider.LITELLM, {}) is False + + +def test_litellm_is_keyless_at_fast_agent_layer(monkeypatch: pytest.MonkeyPatch) -> None: + """LiteLLM resolves backing-provider creds itself; no LITELLM_API_KEY required.""" + from fast_agent.llm import provider_key_manager as pkm + + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + assert "litellm" in pkm.API_KEYLESS_PROVIDERS + # api_key resolves to empty string when nothing is configured (does not raise) + assert ProviderKeyManager.get_api_key("litellm", {}) == "" + + +def test_litellm_picks_up_proxy_api_key_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_API_KEY", "proxy-key-xyz") + assert ProviderKeyManager.get_api_key("litellm", {}) == "proxy-key-xyz" + + +def test_litellm_picks_up_proxy_api_key_from_config() -> None: + config = {"litellm": {"api_key": "config-key-abc"}} + assert ProviderKeyManager.get_api_key("litellm", config) == "config-key-abc" + + +def test_factory_dispatches_to_litellm_class() -> None: + cls = ModelFactory._load_provider_class(Provider.LITELLM) # noqa: SLF001 + from fast_agent.llm.provider.litellm.llm_litellm import LiteLLMLLM + + assert cls is LiteLLMLLM + + +def test_model_spec_parses_into_litellm_provider_and_path() -> None: + parsed = ModelFactory.parse_model_string("litellm.anthropic/claude-sonnet-4-5") + assert parsed.provider is Provider.LITELLM + assert parsed.model_name == "anthropic/claude-sonnet-4-5" + + +def test_model_spec_parses_with_colons_and_at() -> None: + """Bedrock and Vertex specs include `:` and `@` characters.""" + parsed = ModelFactory.parse_model_string( + "litellm.bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + assert parsed.provider is Provider.LITELLM + assert parsed.model_name == "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0" + + parsed = ModelFactory.parse_model_string("litellm.vertex_ai/claude-sonnet-4-5@20250929") + assert parsed.provider is Provider.LITELLM + assert parsed.model_name == "vertex_ai/claude-sonnet-4-5@20250929" + + +def test_catalog_adapter_returns_full_litellm_inventory() -> None: + adapter = LiteLLMModelCatalogAdapter() + inventory = adapter.discover({}) + # Real LiteLLM SDK has 80+ provider keys and 1000+ models + assert len(inventory.all_models) > 1000 + assert inventory.current_models == inventory.all_models + assert all(spec.startswith("litellm.") for spec in inventory.all_models) + + +def test_catalog_adapter_collapses_redundant_provider_prefix() -> None: + """When a model in `models_by_provider[X]` already starts with `X/`, our + spec should collapse it to a single `litellm.X/...` rather than emitting + `litellm.X/X/...`. + + Note: LiteLLM's data legitimately contains nested specs like + `openrouter/openrouter/auto` (where the second `openrouter/` is part of + the OpenRouter model name itself). Those are valid and must be preserved + when round-tripping into `litellm.acompletion(model=...)`. + """ + adapter = LiteLLMModelCatalogAdapter() + inventory = adapter.discover({}) + + # Common, well-known specs should round-trip cleanly without redundant prefixes + expected = { + "litellm.openai/gpt-4o", + "litellm.anthropic/claude-sonnet-4-5", + "litellm.gemini/gemini-2.5-pro", + "litellm.groq/llama-3.3-70b-versatile", + } + seen = set(inventory.all_models) + missing = expected - seen + assert missing == set(), f"Expected popular LiteLLM specs missing: {missing}" + + +def test_catalog_adapter_returns_empty_when_litellm_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import builtins + + real_import = builtins.__import__ + + def _no_litellm(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "litellm": + raise ImportError("simulated") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_litellm) + adapter = LiteLLMModelCatalogAdapter() + inventory = adapter.discover({}) + assert inventory.all_models == () + assert inventory.current_models == () + + +def test_registry_dispatches_to_litellm_adapter() -> None: + inventory = ProviderModelCatalogRegistry.discover(Provider.LITELLM, {}) + assert len(inventory.all_models) > 1000 + + +def test_curated_entries_present_in_picker_snapshot() -> None: + """Wizard snapshot should include LiteLLM with at least 10 curated entries.""" + from fast_agent.ui.model_picker_common import build_snapshot, find_provider + + snapshot = build_snapshot(config_path=None) + option = find_provider(snapshot, "litellm") + assert option.active is True + assert len(option.curated_entries) >= 10 + + +def test_per_model_backing_available_flag(monkeypatch: pytest.MonkeyPatch) -> None: + """Each LiteLLM curated row should reflect whether its backing provider has creds. + + This is what makes the wizard's ✓/✗ markers meaningful: a user without + `OPENAI_API_KEY` set should see ✗ next to LiteLLM's OpenAI rows even + though the LiteLLM SDK itself is installed. + """ + from fast_agent.ui.model_picker_common import ( + build_snapshot, + find_provider, + model_options_for_option, + ) + + # Strip every *_API_KEY and *_BASE_URL so all backings start unconfigured + for key in list(__import__("os").environ): + if key.endswith("_API_KEY") or key.endswith("_BASE_URL"): + monkeypatch.delenv(key, raising=False) + + snapshot = build_snapshot(config_path=None) + option = find_provider(snapshot, "litellm") + models = {m.spec: m for m in model_options_for_option(snapshot, option, source="curated")} + + # No creds: all should be False (LiteLLM-known) or None (unknown spec) + openai_row = models["litellm.openai/gpt-4o"] + assert openai_row.backing_available is False + anthropic_row = models["litellm.anthropic/claude-sonnet-4-6"] + assert anthropic_row.backing_available is False + + # Set ANTHROPIC_API_KEY -> only the anthropic rows flip to True + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + snapshot2 = build_snapshot(config_path=None) + option2 = find_provider(snapshot2, "litellm") + models2 = {m.spec: m for m in model_options_for_option(snapshot2, option2, source="curated")} + + assert models2["litellm.anthropic/claude-sonnet-4-6"].backing_available is True + assert models2["litellm.openai/gpt-4o"].backing_available is False + + +def test_litellm_backing_creds_present_returns_none_for_non_litellm_spec() -> None: + from fast_agent.ui.model_picker_common import litellm_backing_creds_present + + assert litellm_backing_creds_present("openai.gpt-4o") is None + assert litellm_backing_creds_present("anthropic.claude-sonnet-4-5") is None + + +def test_all_scope_includes_dynamic_catalog() -> None: + """The picker's `all` scope should pull in LiteLLM's full registry, not just curated.""" + from fast_agent.ui.model_picker_common import ( + build_snapshot, + find_provider, + model_options_for_option, + ) + + snapshot = build_snapshot(config_path=None) + option = find_provider(snapshot, "litellm") + curated = model_options_for_option(snapshot, option, source="curated") + full = model_options_for_option(snapshot, option, source="all") + assert len(full) > len(curated) + 100 + # Curated entries are still first + curated_specs = {opt.spec for opt in curated} + assert curated_specs.issubset({opt.spec for opt in full}) + + +# --- Provider class shape tests -------------------------------------------------- + + +def test_litellm_client_shim_is_async_context_manager() -> None: + from fast_agent.llm.provider.litellm.llm_litellm import _LiteLLMClientShim + + shim = _LiteLLMClientShim( + api_key=None, + base_url=None, + default_headers=None, + drop_params=True, + timeout=None, + ) + assert hasattr(shim, "__aenter__") + assert hasattr(shim, "__aexit__") + assert hasattr(shim.chat, "completions") + assert callable(shim.chat.completions.create) + + +@pytest.mark.asyncio +async def test_shim_forwards_api_base_and_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + """The shim must pass `api_base`, `api_key`, `timeout`, and `drop_params` to litellm.acompletion.""" + from fast_agent.llm.provider.litellm.llm_litellm import _LiteLLMClientShim + + captured: dict[str, Any] = {} + + async def _fake_acompletion(**kwargs: Any) -> Any: + captured.update(kwargs) + return MagicMock() + + import litellm + + monkeypatch.setattr(litellm, "acompletion", _fake_acompletion) + + shim = _LiteLLMClientShim( + api_key="proxy-key", + base_url="http://localhost:4000", + default_headers={"X-Trace": "abc"}, + drop_params=True, + timeout=45, + ) + await shim.chat.completions.create( + model="anthropic/claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + + assert captured["api_key"] == "proxy-key" + assert captured["api_base"] == "http://localhost:4000" + assert captured["extra_headers"] == {"X-Trace": "abc"} + assert captured["timeout"] == 45 + assert captured["drop_params"] is True + assert captured["stream"] is True + assert captured["model"] == "anthropic/claude-sonnet-4-5" + + +def test_config_bridges_anthropic_api_key_to_env(monkeypatch: pytest.MonkeyPatch) -> None: + """`anthropic.api_key` in fastagent.config.yaml should be exported as + ANTHROPIC_API_KEY so LiteLLM's anthropic backing picks it up at call time. + """ + from fast_agent.llm.provider.litellm.llm_litellm import ( + _bridge_fastagent_config_to_litellm_env, + ) + + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + class _Section: + def __init__(self, **fields: Any) -> None: + for key, value in fields.items(): + setattr(self, key, value) + + class _FakeConfig: + anthropic = _Section(api_key="cfg-anthropic-key", base_url=None) + openai = _Section(api_key=None, base_url=None) + + _bridge_fastagent_config_to_litellm_env(_FakeConfig()) + + assert os.environ.get("ANTHROPIC_API_KEY") == "cfg-anthropic-key" + assert os.environ.get("OPENAI_API_KEY") is None + + +def test_config_bridge_does_not_overwrite_existing_env(monkeypatch: pytest.MonkeyPatch) -> None: + """User-exported env vars must take precedence over the config file.""" + from fast_agent.llm.provider.litellm.llm_litellm import ( + _bridge_fastagent_config_to_litellm_env, + ) + + monkeypatch.setenv("ANTHROPIC_API_KEY", "env-wins") + + class _Section: + api_key = "cfg-loses" + base_url = None + + class _FakeConfig: + anthropic = _Section() + + _bridge_fastagent_config_to_litellm_env(_FakeConfig()) + + assert os.environ.get("ANTHROPIC_API_KEY") == "env-wins" + + +def test_config_bridge_handles_none_config() -> None: + from fast_agent.llm.provider.litellm.llm_litellm import ( + _bridge_fastagent_config_to_litellm_env, + ) + + # Should not raise when config is None + _bridge_fastagent_config_to_litellm_env(None) + + +@pytest.mark.asyncio +async def test_shim_does_not_overwrite_explicit_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: + """If the caller passes `timeout` or `api_key`, the shim must not override it.""" + from fast_agent.llm.provider.litellm.llm_litellm import _LiteLLMClientShim + + captured: dict[str, Any] = {} + + async def _fake_acompletion(**kwargs: Any) -> Any: + captured.update(kwargs) + return MagicMock() + + import litellm + + monkeypatch.setattr(litellm, "acompletion", _fake_acompletion) + + shim = _LiteLLMClientShim( + api_key="shim-key", + base_url="http://shim", + default_headers=None, + drop_params=False, + timeout=10, + ) + await shim.chat.completions.create( + model="openai/gpt-4o", + messages=[], + api_key="caller-key", + timeout=99, + ) + + assert captured["api_key"] == "caller-key" + assert captured["timeout"] == 99 + assert captured["api_base"] == "http://shim" + assert captured["drop_params"] is False diff --git a/tests/unit/fast_agent/llm/test_model_factory.py b/tests/unit/fast_agent/llm/test_model_factory.py index 515f7e8a8..e1c5cbd74 100644 --- a/tests/unit/fast_agent/llm/test_model_factory.py +++ b/tests/unit/fast_agent/llm/test_model_factory.py @@ -22,7 +22,6 @@ from fast_agent.llm.model_factory import ModelFactory, ParsedModelSpec, Provider from fast_agent.llm.model_selection import ModelSelectionCatalog from fast_agent.llm.provider.anthropic.llm_anthropic import AnthropicLLM -from fast_agent.llm.provider.anthropic.llm_anthropic_vertex import AnthropicVertexLLM from fast_agent.llm.provider.openai.llm_generic import GenericLLM from fast_agent.llm.provider.openai.llm_huggingface import HuggingFaceLLM from fast_agent.llm.provider.openai.llm_openai import OpenAILLM @@ -35,7 +34,7 @@ TEST_ALIASES = { "kimi": "hf.moonshotai/Kimi-K2-Instruct-0905", # No default provider "glm": "hf.zai-org/GLM-4.6:cerebras", # Has default provider - "qwen35": "hf.Qwen/Qwen3.5-397B-A17B:novita", + "qwen3": "hf.Qwen/Qwen3-Next-80B-A3B-Instruct:together", "minimax": "hf.MiniMaxAI/MiniMax-M2", # No default provider } @@ -44,8 +43,8 @@ def test_simple_model_names(): """Test parsing of simple model names""" cases = [ ("o1-mini", Provider.RESPONSES), - ("claude-haiku-4-5", Provider.ANTHROPIC), - ("claude-sonnet-4-6", Provider.ANTHROPIC), + ("claude-3-haiku-20240307", Provider.ANTHROPIC), + ("claude-3-5-sonnet-20240620", Provider.ANTHROPIC), ("claude-opus-4-6", Provider.ANTHROPIC), ] @@ -60,9 +59,9 @@ def test_full_model_strings(): """Test parsing of full model strings with providers""" cases = [ ( - "anthropic.claude-haiku-4-5", + "anthropic.claude-3-haiku-20240307", Provider.ANTHROPIC, - "claude-haiku-4-5", + "claude-3-haiku-20240307", None, ), ("openai.gpt-4.1", Provider.OPENAI, "gpt-4.1", None), @@ -95,9 +94,6 @@ def test_deprecated_reasoning_suffix_is_rejected() -> None: with pytest.raises(ModelConfigError, match=r"Use '\?reasoning=' instead"): ModelFactory.parse_model_string("openai/o1.high") - with pytest.raises(ModelConfigError, match=r"Use '\?reasoning=' instead"): - ModelFactory.parse_model_string("openai.o1.HIGH") - def test_model_query_reasoning_effort(): config = ModelFactory.parse_model_string("openai.o1?reasoning=low") @@ -136,11 +132,6 @@ def test_model_query_instant_mode_toggle(): assert config.reasoning_effort == ReasoningEffortSetting(kind="toggle", value=True) -def test_model_query_instant_rejects_reasoning_only_value(): - with pytest.raises(ModelConfigError, match="Invalid instant query value"): - ModelFactory.parse_model_string("hf.moonshotai/Kimi-K2.5?instant=auto") - - def test_model_query_structured_json(): config = ModelFactory.parse_model_string("claude-sonnet-4-5?structured=json") assert config.provider == Provider.ANTHROPIC @@ -155,20 +146,8 @@ def test_model_query_structured_tool_use(): assert config.structured_output_mode == "tool_use" -def test_model_query_structured_tools_policy(): - config = ModelFactory.parse_model_string( - "claude-sonnet-4-6?structured=json&structured_tools=defer" - ) - assert config.structured_tool_policy == "defer" - - camel_case_config = ModelFactory.parse_model_string( - "claude-sonnet-4-6?structuredToolPolicy=%20DEFER%20" - ) - assert camel_case_config.structured_tool_policy == "defer" - - def test_model_query_unknown_parameter_is_rejected() -> None: - with pytest.raises(ModelConfigError, match="Unsupported model query parameter 'routing'"): + with pytest.raises(ModelConfigError, match="Unsupported model query parameter"): ModelFactory.parse_model_string("claude-sonnet-4-6?routing=vertex") @@ -180,28 +159,10 @@ def test_explicit_anthropic_vertex_provider_namespace() -> None: def test_model_query_unknown_parameter_rejected_for_non_anthropic_model(): - with pytest.raises(ModelConfigError, match="Unsupported model query parameter 'routing'"): + with pytest.raises(ModelConfigError, match="Unsupported model query parameter"): ModelFactory.parse_model_string("openai.gpt-4.1?routing=vertex") -def test_model_query_multiple_unknown_parameters_uses_plural_error() -> None: - with pytest.raises( - ModelConfigError, - match=r"Unsupported model query parameters 'other', 'routing'", - ): - ModelFactory.parse_model_string("openai.gpt-4.1?routing=vertex&other=true") - - -def test_model_query_rejects_blank_unknown_parameter() -> None: - with pytest.raises(ModelConfigError, match="Unsupported model query parameter 'unknown'"): - ModelFactory.parse_model_string("gpt-5?unknown=") - - -def test_model_preset_query_rejects_blank_values() -> None: - with pytest.raises(ModelConfigError): - ModelFactory.parse_model_string("broken", presets={"broken": "gpt-5?temperature="}) - - def test_model_query_text_verbosity(): config = ModelFactory.parse_model_string("gpt-5?verbosity=med&reasoning=high") assert config.provider == Provider.RESPONSES @@ -284,7 +245,7 @@ def test_kimi25_alias_sets_thinking_sampling_defaults() -> None: config = ModelFactory.parse_model_string("kimi25") assert config.provider == Provider.HUGGINGFACE - assert config.model_name == "moonshotai/Kimi-K2.5:novita" + assert config.model_name == "moonshotai/Kimi-K2.5:fireworks-ai" assert config.temperature == 1.0 assert config.top_p == 0.95 assert config.reasoning_effort == ReasoningEffortSetting(kind="toggle", value=True) @@ -294,7 +255,7 @@ def test_kimi25instant_alias_sets_instant_sampling_defaults() -> None: config = ModelFactory.parse_model_string("kimi25instant") assert config.provider == Provider.HUGGINGFACE - assert config.model_name == "moonshotai/Kimi-K2.5:novita" + assert config.model_name == "moonshotai/Kimi-K2.5:fireworks-ai" assert config.temperature == 0.6 assert config.top_p == 0.95 assert config.reasoning_effort == ReasoningEffortSetting(kind="toggle", value=False) @@ -304,17 +265,6 @@ def test_kimi_alias_matches_current_promoted_kimi_defaults() -> None: assert ModelFactory.parse_model_string("kimi") == ModelFactory.parse_model_string("kimi26") -def test_kimithink_alias_maps_to_current_kimi_defaults() -> None: - assert ModelFactory.parse_model_string("kimithink") == ModelFactory.parse_model_string("kimi26") - - -def test_direct_kimi_model_routes_to_huggingface() -> None: - config = ModelFactory.parse_model_string("moonshotai/kimi-k2") - - assert config.provider == Provider.HUGGINGFACE - assert config.model_name == "moonshotai/kimi-k2" - - def test_kimi26_alias_sets_thinking_sampling_defaults() -> None: config = ModelFactory.parse_model_string("kimi26") @@ -352,13 +302,8 @@ def test_model_query_transport_websocket_alias(): assert config.transport == "websocket" -def test_model_query_transport_normalizes_case_and_spacing(): - config = ModelFactory.parse_model_string("codexplan?transport=%20WS%20") - assert config.transport == "websocket" - - def test_model_query_transport_auto(): - config = ModelFactory.parse_model_string("codexplan?transport=auto") + config = ModelFactory.parse_model_string("codexplan52?transport=auto") assert config.transport == "auto" @@ -374,14 +319,9 @@ def test_model_query_service_tier(): assert config.service_tier == "fast" -def test_model_query_service_tier_normalizes_case_and_spacing(): - config = ModelFactory.parse_model_string("responses.gpt-5-mini?service_tier=%20FAST%20") - assert config.service_tier == "fast" - - def test_invalid_service_tier_query(): - with pytest.raises(ModelConfigError, match="service_tier query value: 'turbo'"): - ModelFactory.parse_model_string("responses.gpt-5-mini?service_tier=%20TURBO%20") + with pytest.raises(ModelConfigError): + ModelFactory.parse_model_string("responses.gpt-5-mini?service_tier=turbo") def test_codexresponses_fast_service_tier_query() -> None: @@ -403,10 +343,18 @@ def test_responses_chatgpt_flex_service_tier_query_rejected() -> None: def test_chatgpt_alias_flex_service_tier_query_rejected() -> None: - with pytest.raises(ModelConfigError, match="chat-latest"): + with pytest.raises(ModelConfigError, match="gpt-5.3-chat-latest"): ModelFactory.parse_model_string("chatgpt?service_tier=flex") +def test_responses_codex_52_flex_service_tier_query_allowed() -> None: + config = ModelFactory.parse_model_string("responses.gpt-5.2-codex?service_tier=flex") + + assert config.provider == Provider.RESPONSES + assert config.model_name == "gpt-5.2-codex" + assert config.service_tier == "flex" + + def test_responses_codex_53_flex_service_tier_query_rejected() -> None: with pytest.raises(ModelConfigError, match="gpt-5.3-codex"): ModelFactory.parse_model_string("responses.gpt-5.3-codex?service_tier=flex") @@ -421,7 +369,7 @@ def test_model_query_web_tool_flags(): def test_model_query_web_tool_flags_boolean_aliases(): - config = ModelFactory.parse_model_string("sonnet?web_search=yes&web_fetch=disable") + config = ModelFactory.parse_model_string("sonnet?web_search=true&web_fetch=0") assert config.provider == Provider.ANTHROPIC assert config.model_name == "claude-sonnet-4-6" assert config.web_search is True @@ -444,8 +392,8 @@ def test_invalid_web_tool_query_values(): def test_invalid_transport_query(): - with pytest.raises(ModelConfigError, match="transport query value: 'websock'"): - ModelFactory.parse_model_string("codexplan?transport=%20WEBSOCK%20") + with pytest.raises(ModelConfigError): + ModelFactory.parse_model_string("codexplan?transport=websock") def test_transport_query_allows_responses_default_model(): @@ -481,27 +429,6 @@ def test_transport_query_allows_codexresponses_provider_for_codex_spark(): assert config.transport == "websocket" -def test_transport_query_allows_xai_provider_for_grok(): - config = ModelFactory.parse_model_string("xai.grok-4.3?transport=ws") - assert config.provider == Provider.XAI - assert config.model_name == "grok-4.3" - assert config.transport == "websocket" - - -def test_reasoning_query_allows_xai_grok_43_effort() -> None: - config = ModelFactory.parse_model_string("xai.grok-4.3?reasoning=high") - assert config.provider == Provider.XAI - assert config.model_name == "grok-4.3" - assert config.reasoning_effort == ReasoningEffortSetting(kind="effort", value="high") - - -def test_x_search_query_allows_xai_grok() -> None: - config = ModelFactory.parse_model_string("xai.grok-4.3?x_search=enabled") - assert config.provider == Provider.XAI - assert config.model_name == "grok-4.3" - assert config.x_search is True - - def test_transport_query_rejects_openai_provider_even_with_responses_model(): with pytest.raises(ModelConfigError): ModelFactory.parse_model_string("openai.gpt-5?transport=ws") @@ -529,24 +456,6 @@ def test_factory_passes_transport_to_responses_llm_for_openai_responses_model() assert llm._transport == "websocket" -def test_factory_builds_xai_responses_llm_by_default() -> None: - factory = ModelFactory.create_factory("xai.grok-4.3?transport=ws") - llm = factory(LlmAgent(AgentConfig(name="Test Agent"))) - assert isinstance(llm, ResponsesLLM) - assert llm.provider == Provider.XAI - assert llm._transport == "websocket" - - -def test_factory_passes_x_search_override_to_xai_responses_llm() -> None: - from fast_agent.llm.provider.openai.xai_responses import XAIResponsesLLM - - factory = ModelFactory.create_factory("xai.grok-4.3?x_search=on") - llm = factory(LlmAgent(AgentConfig(name="Test Agent"))) - assert isinstance(llm, XAIResponsesLLM) - assert llm.provider == Provider.XAI - assert llm._x_search_override is True - - def test_factory_passes_service_tier_query_to_request_params() -> None: factory = ModelFactory.create_factory("responses.gpt-5?service_tier=fast") llm = factory(LlmAgent(AgentConfig(name="Test Agent"))) @@ -592,14 +501,6 @@ def test_factory_passes_web_tool_overrides_to_anthropic_llm(): assert llm._web_fetch_override is False -def test_factory_passes_web_search_override_to_anthropic_vertex_llm(): - factory = ModelFactory.create_factory("anthropic-vertex.claude-sonnet-4-6?web_search=on") - llm = factory(LlmAgent(AgentConfig(name="Test Agent"))) - - assert isinstance(llm, AnthropicVertexLLM) - assert llm._web_search_override is True - - def test_factory_passes_web_search_override_to_responses_llm(): factory = ModelFactory.create_factory("responses.gpt-5-mini?web_search=on") llm = factory(LlmAgent(AgentConfig(name="Test Agent"))) @@ -647,17 +548,11 @@ def test_invalid_temperature_query(): ModelFactory.parse_model_string("gpt-5?temperature=hot") -def test_invalid_blank_model_query_values() -> None: - for model_string in ("gpt-5?reasoning=", "gpt-5?temperature="): - with pytest.raises(ModelConfigError): - ModelFactory.parse_model_string(model_string) - - def test_llm_class_creation(): """Test creation of LLM classes""" cases = [ ("gpt-4.1", OpenAILLM), - ("claude-haiku-4-5", AnthropicLLM), + ("claude-3-haiku-20240307", AnthropicLLM), ("openai.gpt-4.1", OpenAILLM), ] @@ -702,15 +597,10 @@ def test_builtin_glm_alias_uses_glm_51_default() -> None: assert legacy.model_name == "zai-org/GLM-5:novita" -def test_opus_alias_resolves_to_current_catalog_model(): - opus_entry = next( - entry - for entry in ModelSelectionCatalog.CATALOG_ENTRIES_BY_PROVIDER[Provider.ANTHROPIC] - if entry.alias == "opus" and entry.current - ) +def test_opus_aliases_resolve_to_opus_47(): config = ModelFactory.parse_model_string("opus") assert config.provider == Provider.ANTHROPIC - assert config.model_name == opus_entry.model + assert config.model_name == "claude-opus-4-7" def test_claude_alias_resolves_to_sonnet_46(): @@ -718,14 +608,6 @@ def test_claude_alias_resolves_to_sonnet_46(): assert config.provider == Provider.ANTHROPIC assert config.model_name == "claude-sonnet-4-6" - config = ModelFactory.parse_model_string("sonnet4") - assert config.provider == Provider.ANTHROPIC - assert config.model_name == "claude-sonnet-4-6" - - config = ModelFactory.parse_model_string("opus4") - assert config.provider == Provider.ANTHROPIC - assert config.model_name == ModelFactory.parse_model_string("opus").model_name - config = ModelFactory.parse_model_string("opus46") assert config.provider == Provider.ANTHROPIC assert config.model_name == "claude-opus-4-6" @@ -740,87 +622,18 @@ def test_gemini31_alias_resolves_to_google_31_preview(): assert config.provider == Provider.GOOGLE assert config.model_name == "gemini-3.1-pro-preview" - config = ModelFactory.parse_model_string("gemini31pro") - assert config.provider == Provider.GOOGLE - assert config.model_name == "gemini-3.1-pro-preview" - - -def test_gemini31_flash_lite_alias_resolves_to_google_preview(): - config = ModelFactory.parse_model_string("gemini3.1flashlite") - assert config.provider == Provider.GOOGLE - assert config.model_name == "gemini-3.1-flash-lite-preview" - - -def test_gemini25_alias_resolves_to_current_google_flash(): - config = ModelFactory.parse_model_string("gemini25") - assert config.provider == Provider.GOOGLE - assert config.model_name == "gemini-2.5-flash" - - -@pytest.mark.parametrize("alias", ["gemini35", "gemini35flash", "gemini3.5flash"]) -def test_gemini35_flash_aliases_resolve_to_current_google_flash(alias: str): - config = ModelFactory.parse_model_string(alias) - assert config.provider == Provider.GOOGLE - assert config.model_name == "gemini-3.5-flash" - - -def test_grok_aliases_resolve_to_xai_grok_43(): - config = ModelFactory.parse_model_string("grok") - assert config.provider == Provider.XAI - assert config.model_name == "grok-4.3" - - config = ModelFactory.parse_model_string("grok4") - assert config.provider == Provider.XAI - assert config.model_name == "grok-4.3" - - -def test_deepseek_alias_resolves_to_direct_deepseek_v4_pro(): - config = ModelFactory.parse_model_string("deepseek") - assert config.provider == Provider.DEEPSEEK - assert config.model_name == "deepseek-v4-pro" - - -def test_deepseek_hf_aliases_resolve_to_hf_deepseek_v4_pro(): - for alias in ("deepseek-hf", "deepseek4-hf", "deepseek4pro-hf", "deepseekv4pro-hf"): - config = ModelFactory.parse_model_string(alias) - assert config.provider == Provider.HUGGINGFACE - assert config.model_name == "deepseek-ai/DeepSeek-V4-Pro:together" - - -def test_deepseek_direct_aliases_resolve_to_official_provider(): - config = ModelFactory.parse_model_string("deepseek-v4-pro") - assert config.provider == Provider.DEEPSEEK - assert config.model_name == "deepseek-v4-pro" - - for alias in ("deepseek4", "deepseek4pro", "deepseekv4pro"): - config = ModelFactory.parse_model_string(alias) - assert config.provider == Provider.DEEPSEEK - assert config.model_name == "deepseek-v4-pro" - - config = ModelFactory.parse_model_string("deepseek4flash") - assert config.provider == Provider.DEEPSEEK - assert config.model_name == "deepseek-v4-flash" - - config = ModelFactory.parse_model_string("deepseek4pro-direct") - assert config.provider == Provider.DEEPSEEK - assert config.model_name == "deepseek-v4-pro" - - -def test_hf_routed_gpt_oss_alias_resolves_model_metadata(): - resolved = ModelFactory.resolve_model_spec("gpt-oss") - - assert resolved.provider == Provider.HUGGINGFACE - assert resolved.wire_model_name == "openai/gpt-oss-120b:cerebras" - assert resolved.max_output_tokens == 32766 - def test_curated_catalog_aliases_are_parseable(): - runtime_presets = ModelFactory.get_runtime_presets() for entry in ModelSelectionCatalog.list_current_entries(): + if "?" in entry.model: + continue if entry.model.startswith("anthropic-vertex."): continue - preset_token = entry.alias.strip() - if runtime_presets.get(preset_token) != entry.model: + # LiteLLM curated aliases (e.g. "gpt-4o", "claude-sonnet") intentionally + # mirror the popular short names users already know; they resolve to + # native providers when typed standalone, and to LiteLLM only when the + # full `litellm./` spec is selected from the picker. + if entry.model.startswith("litellm."): continue alias_config = ModelFactory.parse_model_string(entry.alias) @@ -830,15 +643,6 @@ def test_curated_catalog_aliases_are_parseable(): assert ModelDatabase.normalize_model_name( alias_config.model_name ) == ModelDatabase.normalize_model_name(model_config.model_name) - assert alias_config.reasoning_effort == model_config.reasoning_effort - - -def test_query_backed_catalog_alias_applies_runtime_defaults() -> None: - config = ModelFactory.parse_model_string("gpt-5.5") - - assert config.provider == Provider.RESPONSES - assert config.model_name == "gpt-5.5" - assert config.reasoning_effort == ReasoningEffortSetting(kind="effort", value="medium") def test_codexplan_aliases_use_codex_oauth_provider(): @@ -850,6 +654,10 @@ def test_codexplan_aliases_use_codex_oauth_provider(): assert config.provider == Provider.RESPONSES assert config.model_name == "gpt-5.4" + config = ModelFactory.parse_model_string("codexplan52") + assert config.provider == Provider.CODEX_RESPONSES + assert config.model_name == "gpt-5.2-codex" + config = ModelFactory.parse_model_string("codexspark") assert config.provider == Provider.CODEX_RESPONSES assert config.model_name == "gpt-5.3-codex-spark" @@ -861,10 +669,12 @@ def test_codexplan_aliases_use_codex_oauth_provider(): ("glm", "zai-org/GLM-4.6:cerebras"), ("glm:groq", "zai-org/GLM-4.6:groq"), ("kimi:groq", "moonshotai/Kimi-K2-Instruct-0905:groq"), - ("qwen35:nebius", "Qwen/Qwen3.5-397B-A17B:nebius"), + ("qwen3:nebius", "Qwen/Qwen3-Next-80B-A3B-Instruct:nebius"), ], ) -def test_huggingface_alias_provider_routing_contracts(model: str, expected_model_name: str) -> None: +def test_huggingface_alias_provider_routing_contracts( + model: str, expected_model_name: str +) -> None: """Test HuggingFace alias/provider suffix behavior with stable test aliases.""" config = ModelFactory.parse_model_string(model, presets=TEST_ALIASES) assert config.provider == Provider.HUGGINGFACE @@ -915,14 +725,14 @@ def test_model_query_context_1m_case_insensitive(): def test_model_query_context_invalid_value(): """Only '1m' is accepted; anything else raises.""" - with pytest.raises(ModelConfigError, match="context query value: '2m'"): - ModelFactory.parse_model_string("claude-opus-4-6?context=%202M%20") + with pytest.raises(ModelConfigError): + ModelFactory.parse_model_string("claude-opus-4-6?context=2m") -def test_model_query_context_empty_is_rejected(): - """An explicit context= value is invalid unless it names a supported context.""" - with pytest.raises(ModelConfigError): - ModelFactory.parse_model_string("claude-opus-4-6?context=") +def test_model_query_context_empty_is_ignored(): + """Empty context= is dropped by parse_qs, treated as absent.""" + config = ModelFactory.parse_model_string("claude-opus-4-6?context=") + assert config.long_context is False def test_model_query_context_absent_means_false(): @@ -955,34 +765,6 @@ def test_model_query_task_budget_off_clears_default() -> None: assert config.task_budget_configured is True -def test_model_query_task_budget_aliases_preserve_url_order() -> None: - config = ModelFactory.parse_model_string("claude-opus-4-7?taskBudget=128k&task_budget=off") - - assert config.task_budget_tokens is None - assert config.task_budget_configured is True - - -def test_model_query_alias_families_parse() -> None: - config = ModelFactory.parse_model_string( - "gpt-5?structured_tool_policy=defer&web_search=on&x_search=off" - "&web_fetch=on&taskBudget=20k&temp=0.2&topP=0.9&topK=40" - "&minP=0.1&presencePenalty=0.3&repetitionPenalty=1.1" - ) - - assert config.structured_tool_policy == "defer" - assert config.web_search is True - assert config.x_search is False - assert config.web_fetch is True - assert config.task_budget_tokens == 20_000 - assert config.task_budget_configured is True - assert config.temperature == 0.2 - assert config.top_p == 0.9 - assert config.top_k == 40 - assert config.min_p == 0.1 - assert config.presence_penalty == 0.3 - assert config.repetition_penalty == 1.1 - - def test_model_query_task_budget_rejects_values_below_minimum() -> None: with pytest.raises(ModelConfigError, match="Invalid task_budget query value"): ModelFactory.parse_model_string("claude-opus-4-7?task_budget=10k") @@ -1038,12 +820,6 @@ def test_factory_passes_temperature_query_to_request_params(): assert llm.default_request_params.temperature == 0.42 -def test_model_sampling_query_aliases_preserve_url_order() -> None: - config = ModelFactory.parse_model_string("gpt-5?topP=0.95&top_p=0.25") - - assert config.top_p == 0.25 - - def test_factory_passes_sampling_query_to_request_params() -> None: factory = ModelFactory.create_factory("qwen35") agent = LlmAgent(AgentConfig(name="test")) @@ -1059,20 +835,6 @@ def test_factory_passes_sampling_query_to_request_params() -> None: assert llm.reasoning_effort == ReasoningEffortSetting(kind="toggle", value=True) -def test_factory_sampling_query_overrides_explicit_request_params() -> None: - factory = ModelFactory.create_factory("qwen35") - agent = LlmAgent(AgentConfig(name="test")) - request_params = RequestParams(temperature=0.2, top_p=None, top_k=7) - llm = factory(agent, request_params=request_params) - - assert llm.default_request_params.temperature == 0.6 - assert llm.default_request_params.top_p == 0.95 - assert llm.default_request_params.top_k == 20 - assert llm.default_request_params.min_p == 0.0 - assert llm.default_request_params.presence_penalty == 0.0 - assert llm.default_request_params.repetition_penalty == 1.0 - - def test_hf_sampling_overrides_route_non_openai_fields_to_extra_body() -> None: factory = ModelFactory.create_factory("qwen35") agent = LlmAgent(AgentConfig(name="test")) diff --git a/uv.lock b/uv.lock index df8859a56..84b378e71 100644 --- a/uv.lock +++ b/uv.lock @@ -822,6 +822,7 @@ dependencies = [ all-providers = [ { name = "azure-identity" }, { name = "boto3" }, + { name = "litellm" }, { name = "tensorzero" }, ] azure = [ @@ -833,6 +834,9 @@ batch-parquet = [ bedrock = [ { name = "boto3" }, ] +litellm = [ + { name = "litellm" }, +] parquet = [ { name = "duckdb" }, ] @@ -891,6 +895,8 @@ requires-dist = [ { name = "huggingface-hub", specifier = "==1.18.0" }, { name = "jsonschema", specifier = "==4.26.0" }, { name = "keyring", specifier = "==25.7.0" }, + { name = "litellm", marker = "extra == 'all-providers'", specifier = ">=1.60,<1.85" }, + { name = "litellm", marker = "extra == 'litellm'", specifier = ">=1.60,<1.85" }, { name = "mcp", specifier = "==1.27.2" }, { name = "mslex", specifier = "==1.3.0" }, { name = "multilspy", specifier = "==0.0.15" }, @@ -927,7 +933,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32'", specifier = "==0.22.1" }, { name = "watchfiles", specifier = "==1.2.0" }, ] -provides-extras = ["azure", "bedrock", "tensorzero", "textual", "batch-parquet", "parquet", "privacy", "privacy-gpu", "all-providers"] +provides-extras = ["azure", "bedrock", "tensorzero", "litellm", "textual", "batch-parquet", "parquet", "privacy", "privacy-gpu", "all-providers"] [package.metadata.requires-dev] dev = [ @@ -1025,6 +1031,36 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "fastuuid" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/83/ae12dd39b9a39b55d7f90abb8971f1a5f3c321fd72d5aa83f90dc67fe9ed/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021", size = 510720, upload-time = "2025-10-19T22:42:34.633Z" }, + { url = "https://files.pythonhosted.org/packages/53/b0/a4b03ff5d00f563cc7546b933c28cb3f2a07344b2aec5834e874f7d44143/fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc", size = 262024, upload-time = "2025-10-19T22:30:25.482Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/64aee0a0f6a58eeabadd582e55d0d7d70258ffdd01d093b30c53d668303b/fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5", size = 251679, upload-time = "2025-10-19T22:36:14.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/a7e9cda8369e4f7919d36552db9b2ae21db7915083bc6336f1b0082c8b2e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f", size = 277862, upload-time = "2025-10-19T22:36:23.302Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d3/8ce11827c783affffd5bd4d6378b28eb6cc6d2ddf41474006b8d62e7448e/fastuuid-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87", size = 278278, upload-time = "2025-10-19T22:29:43.809Z" }, + { url = "https://files.pythonhosted.org/packages/a2/51/680fb6352d0bbade04036da46264a8001f74b7484e2fd1f4da9e3db1c666/fastuuid-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b", size = 301788, upload-time = "2025-10-19T22:36:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7c/2014b5785bd8ebdab04ec857635ebd84d5ee4950186a577db9eff0fb8ff6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022", size = 459819, upload-time = "2025-10-19T22:35:31.623Z" }, + { url = "https://files.pythonhosted.org/packages/01/d2/524d4ceeba9160e7a9bc2ea3e8f4ccf1ad78f3bde34090ca0c51f09a5e91/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995", size = 478546, upload-time = "2025-10-19T22:26:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/bc/17/354d04951ce114bf4afc78e27a18cfbd6ee319ab1829c2d5fb5e94063ac6/fastuuid-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab", size = 450921, upload-time = "2025-10-19T22:31:02.151Z" }, + { url = "https://files.pythonhosted.org/packages/fb/be/d7be8670151d16d88f15bb121c5b66cdb5ea6a0c2a362d0dcf30276ade53/fastuuid-0.14.0-cp313-cp313-win32.whl", hash = "sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad", size = 154559, upload-time = "2025-10-19T22:36:36.011Z" }, + { url = "https://files.pythonhosted.org/packages/22/1d/5573ef3624ceb7abf4a46073d3554e37191c868abc3aecd5289a72f9810a/fastuuid-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed", size = 156539, upload-time = "2025-10-19T22:33:35.898Z" }, + { url = "https://files.pythonhosted.org/packages/16/c9/8c7660d1fe3862e3f8acabd9be7fc9ad71eb270f1c65cce9a2b7a31329ab/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad", size = 510600, upload-time = "2025-10-19T22:43:44.17Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f4/a989c82f9a90d0ad995aa957b3e572ebef163c5299823b4027986f133dfb/fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b", size = 262069, upload-time = "2025-10-19T22:43:38.38Z" }, + { url = "https://files.pythonhosted.org/packages/da/6c/a1a24f73574ac995482b1326cf7ab41301af0fabaa3e37eeb6b3df00e6e2/fastuuid-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714", size = 251543, upload-time = "2025-10-19T22:32:22.537Z" }, + { url = "https://files.pythonhosted.org/packages/1a/20/2a9b59185ba7a6c7b37808431477c2d739fcbdabbf63e00243e37bd6bf49/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f", size = 277798, upload-time = "2025-10-19T22:33:53.821Z" }, + { url = "https://files.pythonhosted.org/packages/ef/33/4105ca574f6ded0af6a797d39add041bcfb468a1255fbbe82fcb6f592da2/fastuuid-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f", size = 278283, upload-time = "2025-10-19T22:29:02.812Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8c/fca59f8e21c4deb013f574eae05723737ddb1d2937ce87cb2a5d20992dc3/fastuuid-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75", size = 301627, upload-time = "2025-10-19T22:35:54.985Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e2/f78c271b909c034d429218f2798ca4e89eeda7983f4257d7865976ddbb6c/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4", size = 459778, upload-time = "2025-10-19T22:28:00.999Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f0/5ff209d865897667a2ff3e7a572267a9ced8f7313919f6d6043aed8b1caa/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad", size = 478605, upload-time = "2025-10-19T22:36:21.764Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c8/2ce1c78f983a2c4987ea865d9516dbdfb141a120fd3abb977ae6f02ba7ca/fastuuid-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8", size = 450837, upload-time = "2025-10-19T22:34:37.178Z" }, + { url = "https://files.pythonhosted.org/packages/df/60/dad662ec9a33b4a5fe44f60699258da64172c39bd041da2994422cdc40fe/fastuuid-0.14.0-cp314-cp314-win32.whl", hash = "sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06", size = 154532, upload-time = "2025-10-19T22:35:18.217Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/da4db31001e854025ffd26bc9ba0740a9cbba2c3259695f7c5834908b336/fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a", size = 156457, upload-time = "2025-10-19T22:33:44.579Z" }, +] + [[package]] name = "filelock" version = "3.29.1" @@ -1347,14 +1383,14 @@ wheels = [ [[package]] name = "importlib-metadata" -version = "9.0.0" +version = "8.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/72/c600ae4f68c28fc19f9c31b9403053e5dbb8cace2e6842c7b7c3e4d42fe9/importlib_metadata-8.9.0.tar.gz", hash = "sha256:58850626cef4bd2df100378b0f2aea9724a7b92f10770d547725b047078f99ee", size = 56140, upload-time = "2026-03-20T16:56:26.362Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, ] [[package]] @@ -1659,6 +1695,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, ] +[[package]] +name = "litellm" +version = "1.84.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/5c/c2f32e76433515fa14be658238ac1a6bda6de8bc7da320bf27ab6a5321a3/litellm-1.84.6.tar.gz", hash = "sha256:a58f300e8d9a0579152d3df30b722c1eecc4fe0dbf0b3577dfa53e26e18b58ce", size = 15109608, upload-time = "2026-06-09T01:36:49.394Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/fb/43df8bb9d72b420fdcb49b137f695a5bba21cf958e529eb4918d1bc20b83/litellm-1.84.6-py3-none-any.whl", hash = "sha256:4f194a9e3265568a4838742d65cdfb03d483bc0ba7ebdf1a119a4557647d2ed4", size = 16740650, upload-time = "2026-06-09T01:36:45.626Z" }, +] + [[package]] name = "lsprotocol" version = "2023.0.1"