diff --git a/src/Metadata/GoogleModelMetadataDirectory.php b/src/Metadata/GoogleModelMetadataDirectory.php index 5fef43b..082f3bc 100644 --- a/src/Metadata/GoogleModelMetadataDirectory.php +++ b/src/Metadata/GoogleModelMetadataDirectory.php @@ -181,6 +181,18 @@ protected function parseResponseToModelMetadataList(Response $response): array new SupportedOption(OptionEnum::outputMediaAspectRatio(), $geminiImageAspectRatios), new SupportedOption(OptionEnum::customOptions()), ]; + $ttsCapabilities = [ + CapabilityEnum::textToSpeechConversion(), + ]; + $ttsOptions = [ + new SupportedOption(OptionEnum::inputModalities(), [[ModalityEnum::text()]]), + new SupportedOption(OptionEnum::outputModalities(), [[ModalityEnum::audio()]]), + // Gemini TTS returns raw PCM which we wrap into a WAV container. + new SupportedOption(OptionEnum::outputMimeType(), ['audio/wav']), + new SupportedOption(OptionEnum::outputFileType(), [FileTypeEnum::inline()]), + new SupportedOption(OptionEnum::outputSpeechVoice()), + new SupportedOption(OptionEnum::customOptions()), + ]; $modelsData = (array) $responseData['models']; @@ -193,13 +205,21 @@ static function (array $modelData) use ( $geminiMultimodalImageOutputOptions, $imagenCapabilities, $imagenOptions, + $ttsCapabilities, + $ttsOptions, $gemini31ImageAspectRatios ): ModelMetadata { $modelId = $modelData['baseModelId'] ?? $modelData['name']; if (str_starts_with($modelId, 'models/')) { $modelId = substr($modelId, 7); } - if ( + if (str_contains($modelId, '-tts')) { + // TTS models (e.g. gemini-2.5-flash-preview-tts, + // gemini-3.1-flash-tts-preview) also report generateContent, + // so this check must come first. + $modelCaps = $ttsCapabilities; + $modelOptions = $ttsOptions; + } elseif ( isset($modelData['supportedGenerationMethods']) && is_array($modelData['supportedGenerationMethods']) && in_array('generateContent', $modelData['supportedGenerationMethods'], true) diff --git a/src/Models/GoogleTextToSpeechConversionModel.php b/src/Models/GoogleTextToSpeechConversionModel.php new file mode 100644 index 0000000..08a1ece --- /dev/null +++ b/src/Models/GoogleTextToSpeechConversionModel.php @@ -0,0 +1,402 @@ +} + * @phpstan-type CandidateData array{content?: MessageData} + * @phpstan-type UsageData array{ + * promptTokenCount?: int, + * candidatesTokenCount?: int, + * totalTokenCount?: int + * } + * @phpstan-type ResponseData array{ + * candidates?: list, + * usageMetadata?: UsageData + * } + * @phpstan-type InlineAudioData array{data: string, mimeType: string} + */ +class GoogleTextToSpeechConversionModel extends AbstractApiBasedModel implements + TextToSpeechConversionModelInterface +{ + /** + * Default prebuilt Gemini voice used when none is configured. + */ + private const DEFAULT_VOICE = 'Kore'; + + /** + * Default PCM sample rate (Hz) assumed when the response MIME omits it. + */ + private const DEFAULT_SAMPLE_RATE = 24000; + + /** + * {@inheritDoc} + * + * Since we call the Google API, the API key must be wrapped in the Google + * specific authentication class. + * + * @since n.e.x.t + */ + public function getRequestAuthentication(): RequestAuthenticationInterface + { + $requestAuthentication = parent::getRequestAuthentication(); + if (!$requestAuthentication instanceof ApiKeyRequestAuthentication) { + return $requestAuthentication; + } + return new GoogleApiKeyRequestAuthentication($requestAuthentication->getApiKey()); + } + + /** + * {@inheritDoc} + * + * @since n.e.x.t + */ + public function convertTextToSpeechResult(array $prompt): GenerativeAiResult + { + $httpTransporter = $this->getHttpTransporter(); + + $params = $this->prepareConvertParams($prompt); + + $request = new Request( + HttpMethodEnum::POST(), + GoogleProvider::url("models/{$this->metadata()->getId()}:generateContent"), + ['Content-Type' => 'application/json'], + $params, + $this->getRequestOptions() + ); + + // Add authentication credentials to the request. + $request = $this->getRequestAuthentication()->authenticateRequest($request); + + // Send and process the request. + $response = $httpTransporter->send($request); + ResponseUtil::throwIfNotSuccessful($response); + + return $this->parseResponseToGenerativeAiResult($response); + } + + /** + * Prepares the request parameters for the Gemini generateContent endpoint. + * + * @since n.e.x.t + * + * @param list $prompt The prompt messages containing the text. + * @return array The parameters for the API request. + */ + protected function prepareConvertParams(array $prompt): array + { + $config = $this->getConfig(); + + $generationConfig = [ + 'responseModalities' => ['AUDIO'], + 'speechConfig' => [ + 'voiceConfig' => [ + 'prebuiltVoiceConfig' => [ + 'voiceName' => $this->prepareVoice(), + ], + ], + ], + ]; + + $params = [ + 'contents' => [ + [ + 'parts' => [ + ['text' => $this->preparePromptText($prompt)], + ], + ], + ], + ]; + + /* + * Custom options are merged into generationConfig when prefixed with + * "generationConfig.", otherwise at the top level. This mirrors the + * behavior of GoogleTextGenerationModel. The generationConfig is built + * as a separate array and assigned last to keep offset access typed. + */ + $customOptions = $config->getCustomOptions(); + foreach ($customOptions as $key => $value) { + if (str_starts_with($key, 'generationConfig.')) { + $subKey = (string) substr($key, strlen('generationConfig.')); + if (isset($generationConfig[$subKey])) { + throw new InvalidArgumentException( + sprintf( + 'The custom generationConfig option "%s" conflicts with an existing parameter.', + $subKey + ) + ); + } + $generationConfig[$subKey] = $value; + continue; + } + + if (isset($params[$key])) { + throw new InvalidArgumentException( + sprintf('The custom option "%s" conflicts with an existing parameter.', $key) + ); + } + $params[$key] = $value; + } + + $params['generationConfig'] = $generationConfig; + + return $params; + } + + /** + * Extracts the plain text to synthesize from the prompt. + * + * @since n.e.x.t + * + * @param list $prompt The prompt messages. + * @return string The text to convert to speech. + * @throws InvalidArgumentException If no text is found in the prompt. + */ + protected function preparePromptText(array $prompt): string + { + $text = ''; + foreach ($prompt as $message) { + foreach ($message->getParts() as $part) { + $partText = $part->getText(); + if ($partText !== null) { + $text .= ('' === $text ? '' : "\n") . $partText; + } + } + } + + if ('' === $text) { + throw new InvalidArgumentException( + 'The prompt must contain text to convert to speech.' + ); + } + + return $text; + } + + /** + * Resolves the voice to use, falling back to the default. + * + * @since n.e.x.t + * + * @return string The prebuilt voice name. + */ + protected function prepareVoice(): string + { + $voice = $this->getConfig()->getOutputSpeechVoice(); + + return ($voice === null || '' === $voice) ? self::DEFAULT_VOICE : $voice; + } + + /** + * Parses the response, wrapping the returned PCM audio into a WAV file. + * + * @since n.e.x.t + * + * @param Response $response The HTTP response. + * @return GenerativeAiResult The generative AI result containing the audio file. + * @throws ResponseException If the response is missing the expected audio data. + */ + protected function parseResponseToGenerativeAiResult(Response $response): GenerativeAiResult + { + /** @var ResponseData|null $data */ + $data = $response->getData(); + + $inlineData = $this->extractInlineAudioData($data); + + $pcm = base64_decode($inlineData['data'], true); + if ($pcm === false) { + throw ResponseException::fromInvalidData( + $this->providerMetadata()->getName(), + 'candidates[0].content.parts[0].inlineData.data', + 'The audio data could not be base64-decoded.' + ); + } + + $sampleRate = $this->parseSampleRate($inlineData['mimeType']); + $wav = $this->wrapPcmInWav($pcm, $sampleRate); + + $audioFile = new File(base64_encode($wav), 'audio/wav'); + + $message = new Message(MessageRoleEnum::model(), [new MessagePart($audioFile)]); + $candidate = new Candidate($message, FinishReasonEnum::stop()); + + $tokenUsage = $this->parseTokenUsage($data); + + return new GenerativeAiResult( + $this->generateResultId(), + [$candidate], + $tokenUsage, + $this->providerMetadata(), + $this->metadata() + ); + } + + /** + * Extracts the inline audio data (base64 + MIME) from the response. + * + * @since n.e.x.t + * + * @param ResponseData|null $data The decoded response data. + * @return InlineAudioData The inline audio data. + * @throws ResponseException If the audio part is missing or malformed. + */ + protected function extractInlineAudioData(?array $data): array + { + $parts = $data['candidates'][0]['content']['parts'] ?? null; + if (!is_array($parts)) { + throw ResponseException::fromMissingData( + $this->providerMetadata()->getName(), + 'candidates[0].content.parts' + ); + } + + foreach ($parts as $part) { + if ( + is_array($part) && + isset($part['inlineData']['data']) && + is_string($part['inlineData']['data']) + ) { + $mimeType = isset($part['inlineData']['mimeType']) && is_string($part['inlineData']['mimeType']) + ? $part['inlineData']['mimeType'] + : 'audio/L16;codec=pcm;rate=' . self::DEFAULT_SAMPLE_RATE; + + return [ + 'data' => $part['inlineData']['data'], + 'mimeType' => $mimeType, + ]; + } + } + + throw ResponseException::fromMissingData( + $this->providerMetadata()->getName(), + 'candidates[0].content.parts[].inlineData' + ); + } + + /** + * Parses the PCM sample rate from a Gemini audio MIME type. + * + * Example input: "audio/L16;codec=pcm;rate=24000". + * + * @since n.e.x.t + * + * @param string $mimeType The Gemini inline audio MIME type. + * @return int The sample rate in Hz. + */ + protected function parseSampleRate(string $mimeType): int + { + if (preg_match('/rate=(\d+)/', $mimeType, $matches)) { + return (int) $matches[1]; + } + + return self::DEFAULT_SAMPLE_RATE; + } + + /** + * Wraps raw signed 16-bit little-endian PCM data in a WAV container. + * + * @since n.e.x.t + * + * @param string $pcm The raw PCM audio bytes. + * @param int $sampleRate The sample rate in Hz. + * @param int $channels The number of channels. Default 1 (mono). + * @param int $bitsPerSample The bit depth. Default 16. + * @return string The WAV-formatted audio bytes. + */ + protected function wrapPcmInWav( + string $pcm, + int $sampleRate, + int $channels = 1, + int $bitsPerSample = 16 + ): string { + $blockAlign = (int) ($channels * ($bitsPerSample / 8)); + $byteRate = $sampleRate * $blockAlign; + $dataLength = strlen($pcm); + + $header = 'RIFF' + . pack('V', 36 + $dataLength) + . 'WAVE' + . 'fmt ' + . pack('V', 16) // Subchunk1Size for PCM. + . pack('v', 1) // AudioFormat = PCM. + . pack('v', $channels) + . pack('V', $sampleRate) + . pack('V', $byteRate) + . pack('v', $blockAlign) + . pack('v', $bitsPerSample) + . 'data' + . pack('V', $dataLength); + + return $header . $pcm; + } + + /** + * Parses token usage from the response, defaulting to zeros. + * + * @since n.e.x.t + * + * @param ResponseData|null $data The decoded response data. + * @return TokenUsage The token usage. + */ + protected function parseTokenUsage(?array $data): TokenUsage + { + $usage = $data['usageMetadata'] ?? null; + if (!is_array($usage)) { + return new TokenUsage(0, 0, 0); + } + + return new TokenUsage( + isset($usage['promptTokenCount']) && is_int($usage['promptTokenCount']) ? $usage['promptTokenCount'] : 0, + isset($usage['candidatesTokenCount']) && is_int($usage['candidatesTokenCount']) + ? $usage['candidatesTokenCount'] + : 0, + isset($usage['totalTokenCount']) && is_int($usage['totalTokenCount']) ? $usage['totalTokenCount'] : 0 + ); + } + + /** + * Generates a result identifier (the endpoint does not return one for audio). + * + * @since n.e.x.t + * + * @return string The result identifier. + */ + protected function generateResultId(): string + { + return uniqid('google-tts-', true); + } +} diff --git a/src/Provider/GoogleProvider.php b/src/Provider/GoogleProvider.php index 78856a4..f6a1f19 100644 --- a/src/Provider/GoogleProvider.php +++ b/src/Provider/GoogleProvider.php @@ -19,6 +19,7 @@ use WordPress\GoogleAiProvider\Models\GoogleImageGenerationModel; use WordPress\GoogleAiProvider\Models\GoogleTextAndImageGenerationModel; use WordPress\GoogleAiProvider\Models\GoogleTextGenerationModel; +use WordPress\GoogleAiProvider\Models\GoogleTextToSpeechConversionModel; /** * Class for the Google provider. @@ -57,6 +58,9 @@ protected static function createModel( $capabilities = $modelMetadata->getSupportedCapabilities(); foreach ($capabilities as $capability) { + if ($capability->isTextToSpeechConversion()) { + return new GoogleTextToSpeechConversionModel($modelMetadata, $providerMetadata); + } if ($capability->isTextGeneration()) { return new GoogleTextGenerationModel($modelMetadata, $providerMetadata); } @@ -89,9 +93,9 @@ protected static function createProviderMetadata(): ProviderMetadata // For WordPress, we should translate the description. if (function_exists('__')) { // phpcs:ignore Generic.Files.LineLength.TooLong - $providerMetadataArgs[] = __('Text and image generation with Gemini and Imagen.', 'ai-provider-for-google'); + $providerMetadataArgs[] = __('Text, image, and speech generation with Gemini and Imagen.', 'ai-provider-for-google'); } else { - $providerMetadataArgs[] = 'Text and image generation with Gemini and Imagen.'; + $providerMetadataArgs[] = 'Text, image, and speech generation with Gemini and Imagen.'; } } // Provider logoPath support was added in 1.3.0.