diff --git a/.changeset/models-dev-per-model-provider-overrides.md b/.changeset/models-dev-per-model-provider-overrides.md new file mode 100644 index 000000000000..ff2c0833f776 --- /dev/null +++ b/.changeset/models-dev-per-model-provider-overrides.md @@ -0,0 +1,7 @@ +--- +"@mastra/core": minor +--- + +models.dev gateway: honor per-model `provider` overrides (endpoint, request shape, SDK). + +A provider can now serve individual models over a different base URL / request shape than the provider default — e.g. a model served over the OpenAI **Responses** API while the provider default is chat-completions. The models.dev gateway now reads each model's `provider` block (`api`, `shape`, `npm`), so `resolveLanguageModel` routes `shape: "responses"` models via the OpenAI Responses API and `buildUrl` prefers a per-model `api` when present. Providers without per-model overrides are unaffected. diff --git a/packages/core/src/llm/model/gateways/base.ts b/packages/core/src/llm/model/gateways/base.ts index 89fa919857d6..6e9fa98256fd 100644 --- a/packages/core/src/llm/model/gateways/base.ts +++ b/packages/core/src/llm/model/gateways/base.ts @@ -9,6 +9,18 @@ import type { LanguageModelV4 } from '@ai-sdk/provider-v7'; import type { StreamTransport } from '../../../stream/types'; import type { OpenAITransport, ResponsesWebSocketOptions } from '../provider-options.js'; +/** + * Per-model provider override sourced from a models.dev model's `provider` block. + * Lets a single provider serve individual models over a different endpoint / request + * shape / SDK than the provider default (e.g. a model served over the OpenAI + * Responses API while the provider default is chat-completions). + */ +export interface ModelProviderOverride { + api?: string; // Base API URL for this model (may contain ${ENV} templates) + shape?: 'responses' | 'completions'; // Request shape to use for this model + npm?: string; // SDK package to use for this model +} + export interface ProviderConfig { url?: string; apiKeyHeader?: string; @@ -18,6 +30,9 @@ export interface ProviderConfig { docUrl?: string; // Optional documentation URL gateway: string; npm?: string; // NPM package name from models.dev (e.g., "@ai-sdk/anthropic") + // Per-model overrides (endpoint/shape/SDK) keyed by model id, when a provider + // serves some models differently than its default (models.dev model `provider`). + modelOverrides?: Record; } /** diff --git a/packages/core/src/llm/model/gateways/models-dev.test.ts b/packages/core/src/llm/model/gateways/models-dev.test.ts index a447ce365971..6f90bcc762c8 100644 --- a/packages/core/src/llm/model/gateways/models-dev.test.ts +++ b/packages/core/src/llm/model/gateways/models-dev.test.ts @@ -547,6 +547,84 @@ describe('ModelsDevGateway', () => { ); }); + describe('per-model provider overrides', () => { + it('captures a model-level provider override from models.dev into modelOverrides', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + neon: { + id: 'neon', + name: 'Neon', + npm: '@ai-sdk/openai-compatible', + api: '${NEON_AI_GATEWAY_BASE_URL}/ai-gateway/mlflow/v1', + env: ['NEON_AI_GATEWAY_BASE_URL', 'NEON_AI_GATEWAY_TOKEN'], + models: { + 'gpt-5-mini': { + name: 'GPT-5 mini', + provider: { + npm: '@ai-sdk/openai', + api: '${NEON_AI_GATEWAY_BASE_URL}/ai-gateway/openai/v1', + shape: 'responses', + }, + }, + 'claude-haiku-4-5': { name: 'Claude Haiku 4.5' }, + }, + }, + }), + }); + + const providers = await gateway.fetchProviders(); + + expect(providers.neon.modelOverrides).toEqual({ + 'gpt-5-mini': { + npm: '@ai-sdk/openai', + api: '${NEON_AI_GATEWAY_BASE_URL}/ai-gateway/openai/v1', + shape: 'responses', + }, + }); + }); + + const overrideGateway = () => + new ModelsDevGateway({ + neon: { + apiKeyEnvVar: 'NEON_AI_GATEWAY_TOKEN', + name: 'Neon', + models: ['gpt-5-mini', 'claude-haiku-4-5'], + gateway: 'models.dev', + url: 'https://ex.neon.tech/ai-gateway/mlflow/v1', + modelOverrides: { + 'gpt-5-mini': { api: 'https://ex.neon.tech/ai-gateway/openai/v1', shape: 'responses' }, + }, + }, + }); + + it('routes a shape="responses" model via the OpenAI Responses API on a chat-completions provider', async () => { + gateway = overrideGateway(); + + const result = await gateway.resolveLanguageModel({ + providerId: 'neon', + modelId: 'gpt-5-mini', + apiKey: 'nt_live_test', + headers: { 'x-test': 'true' }, + }); + + expect(result).toEqual({ provider: 'openai' }); + expect(createOpenAIMock).toHaveBeenCalledWith({ + apiKey: 'nt_live_test', + baseURL: 'https://ex.neon.tech/ai-gateway/openai/v1', + headers: expect.objectContaining({ 'x-test': 'true' }), + }); + expect(openAIResponsesMock).toHaveBeenCalledWith('gpt-5-mini'); + }); + + it('buildUrl prefers the per-model override endpoint over the provider default', () => { + gateway = overrideGateway(); + + expect(gateway.buildUrl('neon/gpt-5-mini')).toBe('https://ex.neon.tech/ai-gateway/openai/v1'); + expect(gateway.buildUrl('neon/claude-haiku-4-5')).toBe('https://ex.neon.tech/ai-gateway/mlflow/v1'); + }); + }); + describe('integration', () => { it('should handle full flow: fetch, buildUrl, buildHeaders', async () => { mockFetch.mockResolvedValueOnce({ diff --git a/packages/core/src/llm/model/gateways/models-dev.ts b/packages/core/src/llm/model/gateways/models-dev.ts index c5c43d38b199..1febab218e3c 100644 --- a/packages/core/src/llm/model/gateways/models-dev.ts +++ b/packages/core/src/llm/model/gateways/models-dev.ts @@ -15,7 +15,13 @@ import { createGateway } from '@internal/ai-v6'; import { createOpenRouter } from '@openrouter/ai-sdk-provider-v5'; import { parseModelRouterId } from '../gateway-resolver.js'; import { MastraModelGateway } from './base.js'; -import type { AttachmentCapabilities, GatewayLanguageModel, ProviderConfig, TemperatureCapabilities } from './base.js'; +import type { + AttachmentCapabilities, + GatewayLanguageModel, + ModelProviderOverride, + ProviderConfig, + TemperatureCapabilities, +} from './base.js'; import { EXCLUDED_PROVIDERS, MASTRA_USER_AGENT, PROVIDERS_WITH_INSTALLED_PACKAGES } from './constants.js'; interface ModelsDevModelInfo { @@ -25,6 +31,8 @@ interface ModelsDevModelInfo { modalities?: { input?: string[]; output?: string[] }; attachment?: boolean; temperature?: boolean; + // Per-model endpoint/shape/SDK override (models.dev model `provider` block). + provider?: { api?: string; shape?: 'responses' | 'completions'; npm?: string }; [key: string]: unknown; } @@ -173,6 +181,17 @@ export class ModelsDevGateway extends MastraModelGateway { .map(([modelId]) => modelId) .sort(); + // Collect per-model endpoint/shape/SDK overrides. Some providers serve + // individual models over a different endpoint or request shape than the + // provider default (e.g. OpenAI Responses vs chat-completions). + const modelOverrides: Record = {}; + for (const [modelId, modelInfo] of activeModels) { + const ov = modelInfo?.provider; + if (ov && (ov.api || ov.shape || ov.npm)) { + modelOverrides[modelId] = { api: ov.api, shape: ov.shape, npm: ov.npm }; + } + } + // Get the API URL - overrides take priority over models.dev data const url = PROVIDER_OVERRIDES[normalizedId]?.url || providerInfo.api; @@ -211,6 +230,7 @@ export class ModelsDevGateway extends MastraModelGateway { providerInfo.npm !== '@ai-sdk/gateway' ? providerInfo.npm : undefined), + modelOverrides: Object.keys(modelOverrides).length > 0 ? modelOverrides : undefined, }; if (attachmentModels.length > 0) { this.attachmentCapabilities[normalizedId] = attachmentModels; @@ -240,19 +260,24 @@ export class ModelsDevGateway extends MastraModelGateway { } buildUrl(routerId: string, envVars?: typeof process.env): string | undefined { - const { providerId } = parseModelRouterId(routerId); + const { providerId, modelId } = parseModelRouterId(routerId); const config = this.providerConfigs[providerId]; - if (!config?.url) { + // A per-model override may point this model at a different endpoint than the + // provider default; prefer it when present. + const perModelApi = config?.modelOverrides?.[modelId]?.api; + const template = perModelApi ?? config?.url; + + if (!template) { return; } - // Check for custom base URL from env vars + // Check for custom base URL from env vars (explicit override still wins) const baseUrlEnvVar = `${providerId.toUpperCase().replace(/-/g, '_')}_BASE_URL`; const customBaseUrl = envVars?.[baseUrlEnvVar] || process.env[baseUrlEnvVar]; - return customBaseUrl || interpolateUrlTemplate(config.url, envVars); + return customBaseUrl || interpolateUrlTemplate(template, envVars); } getApiKey(modelId: string): Promise { @@ -291,6 +316,14 @@ export class ModelsDevGateway extends MastraModelGateway { const mastraHeaders = { 'User-Agent': MASTRA_USER_AGENT, ...headers }; + // Per-model override: a model may be served over a different request shape + // than its provider default (e.g. the OpenAI Responses API while the provider + // default is chat-completions). Honor `shape` for any provider. + const override = this.providerConfigs[providerId]?.modelOverrides?.[modelId]; + if (override?.shape === 'responses') { + return createOpenAI({ apiKey, baseURL, headers: mastraHeaders }).responses(modelId); + } + switch (providerId) { case 'openai': return createOpenAI({ apiKey, baseURL, headers: mastraHeaders }).responses(modelId); @@ -328,9 +361,10 @@ export class ModelsDevGateway extends MastraModelGateway { return createAnthropic({ apiKey, baseURL, headers: mastraHeaders })(modelId); } default: { - // Check if this provider uses a specific SDK package (e.g., kimi-for-coding uses @ai-sdk/anthropic) + // Check if this provider uses a specific SDK package (e.g., kimi-for-coding uses @ai-sdk/anthropic). + // A per-model override wins over the provider default. const config = this.providerConfigs[providerId]; - const npm = config?.npm; + const npm = override?.npm ?? config?.npm; // Pattern match for any alibaba variant (alibaba, alibaba-cn, alibaba-coding-plan, etc.) if (providerId.includes('alibaba')) {