From 0cbe1502da84fde4fbee65faf0a9a4c5ed738f30 Mon Sep 17 00:00:00 2001 From: priyanshuhaldar007 Date: Thu, 23 Jul 2026 19:48:53 +0530 Subject: [PATCH] feat: add semantic search experiment to AI plugin --- includes/Experiments/Experiments.php | 1 + .../Semantic_Search/Embedding_Api.php | 383 ++++++++++++++++++ .../Semantic_Search/Embedding_Store.php | 166 ++++++++ .../Semantic_Search/Index_Page.php | 266 ++++++++++++ .../Experiments/Semantic_Search/Indexer.php | 131 ++++++ .../List_Table_Integration.php | 148 +++++++ .../Semantic_Search/REST_Controller.php | 215 ++++++++++ .../Semantic_Search/Semantic_Search.php | 270 ++++++++++++ .../Semantic_Search/Vector_Search.php | 177 ++++++++ 9 files changed, 1757 insertions(+) create mode 100644 includes/Experiments/Semantic_Search/Embedding_Api.php create mode 100644 includes/Experiments/Semantic_Search/Embedding_Store.php create mode 100644 includes/Experiments/Semantic_Search/Index_Page.php create mode 100644 includes/Experiments/Semantic_Search/Indexer.php create mode 100644 includes/Experiments/Semantic_Search/List_Table_Integration.php create mode 100644 includes/Experiments/Semantic_Search/REST_Controller.php create mode 100644 includes/Experiments/Semantic_Search/Semantic_Search.php create mode 100644 includes/Experiments/Semantic_Search/Vector_Search.php diff --git a/includes/Experiments/Experiments.php b/includes/Experiments/Experiments.php index 4799075ec..58b70921c 100644 --- a/includes/Experiments/Experiments.php +++ b/includes/Experiments/Experiments.php @@ -27,6 +27,7 @@ final class Experiments { * @var array> */ private const EXPERIMENT_CLASSES = array( // phpcs:ignore SlevomatCodingStandard.Classes.DisallowMultiConstantDefinition -- This is used as an array const. + \WordPress\AI\Experiments\Semantic_Search\Semantic_Search::class, \WordPress\AI\Experiments\Abilities_Explorer\Abilities_Explorer::class, \WordPress\AI\Experiments\AI_Request_Logging\AI_Request_Logging::class, \WordPress\AI\Experiments\Connector_Approval\Connector_Approval::class, diff --git a/includes/Experiments/Semantic_Search/Embedding_Api.php b/includes/Experiments/Semantic_Search/Embedding_Api.php new file mode 100644 index 000000000..e547d61cd --- /dev/null +++ b/includes/Experiments/Semantic_Search/Embedding_Api.php @@ -0,0 +1,383 @@ +generateEmbeddingResult() + * and let the registered connector handle authentication + transport. + * + * @package WordPress\AI\Experiments\Semantic_Search + */ + +declare( strict_types=1 ); + +namespace WordPress\AI\Experiments\Semantic_Search; + +// Exit if accessed directly. +defined( 'ABSPATH' ) || exit; + +/** + * Sends text to an embedding API and returns a float vector. + * + * Supports four providers out of the box: OpenAI, Jina AI, Ollama (local), + * and Google Gemini. Provider selection and credentials are read from the + * namespaced WordPress options registered by the Semantic_Search experiment. + * + * @internal + * @since x.x.x + */ +class Embedding_Api { + + /** + * Per-provider configuration defaults. + * + * Each entry contains: + * - label: Human-readable provider name. + * - default_url: API base URL used when no override is saved. + * - default_model: Model identifier used when no override is saved. + * - dimensions: Output vector length. Source: official model documentation. + * - score_threshold: Cosine similarity cut-off. These are heuristics — no + * authoritative published benchmarks exist. Tune via Settings. + * + * @since x.x.x + * @var array + */ + public const PROVIDERS = array( + 'openai' => array( + 'label' => 'OpenAI', + 'default_url' => 'https://api.openai.com/v1/embeddings', + 'default_model' => 'text-embedding-3-small', + 'dimensions' => 1536, + 'score_threshold' => 0.60, + ), + 'jina' => array( + 'label' => 'Jina AI', + 'default_url' => 'https://api.jina.ai/v1/embeddings', + 'default_model' => 'jina-embeddings-v3', + 'dimensions' => 1024, + 'score_threshold' => 0.55, + ), + 'ollama' => array( + 'label' => 'Ollama (local)', + 'default_url' => 'http://localhost:11434/v1/embeddings', + 'default_model' => 'nomic-embed-text', + 'dimensions' => 768, + 'score_threshold' => 0.50, + ), + 'google' => array( + 'label' => 'Google Gemini', + 'default_url' => 'https://generativelanguage.googleapis.com/v1/models', + 'default_model' => 'gemini-embedding-001', + 'dimensions' => 3072, + 'score_threshold' => 0.45, + ), + ); + + /** + * Active provider key (e.g. 'openai', 'google'). + * + * @since x.x.x + * @var string + */ + private string $provider; + + /** + * API key for the active provider. Empty for Ollama. + * + * @since x.x.x + * @var string + */ + private string $api_key; + + /** + * Embedding model identifier (e.g. 'text-embedding-3-small'). + * + * @since x.x.x + * @var string + */ + private string $model; + + /** + * API base URL for the active provider. + * + * @since x.x.x + * @var string + */ + private string $base_url; + + /** + * Human-readable description of the last API error, or empty string if none. + * + * @since x.x.x + * @var string + */ + private string $last_error = ''; + + /** + * Reads all provider configuration from the saved experiment options. + * + * @since x.x.x + */ + public function __construct() { + $this->provider = (string) get_option( Semantic_Search::get_field_option_name( 'provider' ), 'openai' ); + $this->api_key = (string) get_option( Semantic_Search::get_field_option_name( 'api_key' ), '' ); + $this->model = (string) get_option( Semantic_Search::get_field_option_name( 'model' ), 'text-embedding-3-small' ); + $this->base_url = (string) get_option( Semantic_Search::get_field_option_name( 'base_url' ), 'https://api.openai.com/v1/embeddings' ); + } + + /** + * Returns the active model identifier. + * + * @since x.x.x + * + * @return string Model identifier string (e.g. 'gemini-embedding-001'). + */ + public function get_model(): string { + return $this->model; + } + + /** + * Returns the active provider key. + * + * @since x.x.x + * + * @return string Provider key (e.g. 'openai', 'google'). + */ + public function get_provider(): string { + return $this->provider; + } + + /** + * Returns the human-readable error message from the most recent API call. + * + * Returns an empty string when the last call succeeded. + * + * @since x.x.x + * + * @return string Error description, or empty string on success. + */ + public function get_last_error(): string { + return $this->last_error; + } + + /** + * Returns whether the provider is ready to accept embedding requests. + * + * Ollama requires only a non-empty model name (no key). All other providers + * require a non-empty API key. + * + * @since x.x.x + * + * @return bool True when the minimum required credentials are present. + */ + public function is_configured(): bool { + if ( 'ollama' === $this->provider ) { + return '' !== $this->model; + } + + return '' !== $this->api_key; + } + + /** + * Returns the cosine similarity score threshold for the active provider. + * + * Reads the user-saved option first; falls back to the per-provider default + * from PROVIDERS when the option is empty. An empty string means "use the + * provider default" and is backward-compatible with existing installs that + * predate the user-editable threshold field. + * + * @since x.x.x + * + * @return float Cosine similarity cut-off in the range [0, 1]. + */ + public function get_score_threshold(): float { + $cfg = self::PROVIDERS[ $this->provider ] ?? self::PROVIDERS['openai']; + $saved = (string) get_option( Semantic_Search::get_field_option_name( 'score_threshold' ), '' ); + + return '' !== $saved ? (float) $saved : $cfg['score_threshold']; + } + + /** + * Sends text to the configured embedding provider and returns a float vector. + * + * Dispatches to the Google-specific or OpenAI-compatible implementation + * depending on the active provider. Returns null on any API error; call + * get_last_error() to retrieve the reason. + * + * @since x.x.x + * + * @param string $text The text to embed. + * @return float[]|null Float vector on success, null on failure. + */ + public function generate( string $text ): ?array { + $this->last_error = ''; + + if ( 'google' === $this->provider ) { + return $this->generate_google( $text ); + } + + return $this->generate_openai_compatible( $text ); + } + + /** + * Generates an embedding using the OpenAI-compatible request format. + * + * Used for OpenAI, Jina AI, and Ollama. The Authorization header is omitted + * for Ollama since it requires no API key. + * + * @since x.x.x + * + * @param string $text The text to embed. + * @return float[]|null Float vector on success, null on failure. + */ + private function generate_openai_compatible( string $text ): ?array { + $headers = array( 'Content-Type' => 'application/json' ); + + if ( 'ollama' !== $this->provider ) { + $headers['Authorization'] = 'Bearer ' . $this->api_key; + } + + $response = wp_remote_post( + $this->base_url, + array( + 'timeout' => 30, + 'headers' => $headers, + 'body' => wp_json_encode( + array( + 'input' => $text, + 'model' => $this->model, + ) + ), + ) + ); + + return $this->parse_openai_response( $response ); + } + + /** + * Generates an embedding using the Google Gemini embedContent API. + * + * Google's API differs from the OpenAI-compatible format in three ways: + * - The model name is appended to the URL path, not sent in the body. + * - The API key is passed as a query parameter, not an Authorization header. + * - The request body uses `content.parts[].text` instead of `input`. + * + * @since x.x.x + * + * @param string $text The text to embed. + * @return float[]|null Float vector on success, null on failure. + */ + private function generate_google( string $text ): ?array { + $endpoint = rtrim( $this->base_url, '/' ) . '/' . $this->model . ':embedContent?key=' . $this->api_key; + + $response = wp_remote_post( + $endpoint, + array( + 'timeout' => 30, + 'headers' => array( 'Content-Type' => 'application/json' ), + 'body' => wp_json_encode( + array( + 'content' => array( + 'parts' => array( + array( 'text' => $text ), + ), + ), + ) + ), + ) + ); + + if ( is_wp_error( $response ) ) { + $this->last_error = $response->get_error_message(); + return null; + } + + $code = wp_remote_retrieve_response_code( $response ); + $body = json_decode( wp_remote_retrieve_body( $response ), true ); + + if ( 200 !== $code ) { + $this->last_error = sprintf( 'HTTP %d: %s', $code, $body['error']['message'] ?? 'unknown error' ); + return null; + } + + $values = $body['embedding']['values'] ?? null; + + if ( ! is_array( $values ) ) { + $this->last_error = 'Unexpected response format from Google API.'; + return null; + } + + return array_map( 'floatval', $values ); + } + + /** + * Parses an OpenAI-compatible embedding response into a float vector. + * + * Handles WP_Error from wp_remote_post, non-200 HTTP status codes, and + * unexpected response shapes. On any failure, sets last_error and returns null. + * + * @since x.x.x + * + * @param \WP_Error|array $response Response from wp_remote_post(). + * @return float[]|null Float vector on success, null on failure. + */ + private function parse_openai_response( $response ): ?array { + if ( is_wp_error( $response ) ) { + $this->last_error = $response->get_error_message(); + return null; + } + + $code = wp_remote_retrieve_response_code( $response ); + $body = json_decode( wp_remote_retrieve_body( $response ), true ); + + if ( 200 !== $code ) { + $this->last_error = sprintf( 'HTTP %d: %s', $code, $body['error']['message'] ?? 'unknown error' ); + return null; + } + + $values = $body['data'][0]['embedding'] ?? null; + + if ( ! is_array( $values ) ) { + $this->last_error = 'Unexpected response format from provider.'; + return null; + } + + return array_map( 'floatval', $values ); + } + + /** + * Fetches the names of embedding-capable models available for the current Google API key. + * + * Calls the Google ListModels endpoint and filters to models that support the + * `embedContent` method. Used only by the test-connection REST endpoint to + * surface actionable context when the configured model name returns a 404. + * + * @since x.x.x + * + * @return string[] Model name strings (e.g. ['gemini-embedding-001']), empty on error. + */ + public function list_google_embedding_models(): array { + $endpoint = 'https://generativelanguage.googleapis.com/v1/models?key=' . $this->api_key; + $response = wp_remote_get( $endpoint, array( 'timeout' => 15 ) ); + + if ( is_wp_error( $response ) ) { + return array(); + } + + $body = json_decode( wp_remote_retrieve_body( $response ), true ); + $models = $body['models'] ?? array(); + $names = array(); + + foreach ( $models as $m ) { + $supported = $m['supportedGenerationMethods'] ?? array(); + if ( in_array( 'embedContent', $supported, true ) ) { + $names[] = str_replace( 'models/', '', $m['name'] ?? '' ); + } + } + + return array_filter( $names ); + } +} diff --git a/includes/Experiments/Semantic_Search/Embedding_Store.php b/includes/Experiments/Semantic_Search/Embedding_Store.php new file mode 100644 index 000000000..2a95fa280 --- /dev/null +++ b/includes/Experiments/Semantic_Search/Embedding_Store.php @@ -0,0 +1,166 @@ +get_col( + $wpdb->prepare( + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared + "SELECT p.ID FROM {$wpdb->posts} p + LEFT JOIN {$wpdb->postmeta} pm + ON p.ID = pm.post_id AND pm.meta_key = %s + WHERE p.post_status = 'publish' + AND p.post_type IN ($types_placeholders) + AND pm.meta_value IS NULL + LIMIT %d", + array_merge( + array( self::META_EMBEDDING ), + $post_types, + array( $limit ) + ) + ) + ); + + return array_map( 'intval', $rows ?: array() ); + } + + /** + * Returns total and indexed post/page counts for the current site. + * + * @since x.x.x + * + * @return array{total:int, indexed:int} Counts of total published posts and indexed posts. + */ + public function get_stats(): array { + global $wpdb; + + // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching + $total = (int) $wpdb->get_var( + "SELECT COUNT(*) FROM {$wpdb->posts} + WHERE post_status = 'publish' + AND post_type IN ('post', 'page')" + ); + + // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching + $indexed = (int) $wpdb->get_var( + $wpdb->prepare( + "SELECT COUNT(DISTINCT post_id) FROM {$wpdb->postmeta} + WHERE meta_key = %s", + self::META_EMBEDDING + ) + ); + + return array( + 'total' => $total, + 'indexed' => $indexed, + ); + } + + /** + * Removes the stored embedding and model meta for a post. + * + * @since x.x.x + * + * @param int $post_id Post ID whose index entry should be removed. + * @return void + */ + public function delete( int $post_id ): void { + delete_post_meta( $post_id, self::META_EMBEDDING ); + delete_post_meta( $post_id, self::META_MODEL ); + } +} diff --git a/includes/Experiments/Semantic_Search/Index_Page.php b/includes/Experiments/Semantic_Search/Index_Page.php new file mode 100644 index 000000000..e10d54bef --- /dev/null +++ b/includes/Experiments/Semantic_Search/Index_Page.php @@ -0,0 +1,266 @@ + rest_url( 'ai/v1/semantic-search/index' ), + 'statusUrl' => rest_url( 'ai/v1/semantic-search/index/status' ), + 'testUrl' => rest_url( 'ai/v1/semantic-search/index/test' ), + 'nonce' => wp_create_nonce( 'wp_rest' ), + ) + ); + } + + /** + * Renders the indexing admin page. + * + * Shows a notice with a link to Settings → AI when no embedding provider is + * configured. When a provider is configured, shows the current indexed/total + * counts, a Test Connection button, and an Index All Posts button. All button + * interactions are handled by the inline script at the bottom of the page. + * + * @since x.x.x + * + * @return void + */ + public function render_page(): void { + if ( ! current_user_can( 'manage_options' ) ) { + wp_die( esc_html__( 'You do not have sufficient permissions to access this page.', 'ai' ) ); + } + + $api = new Embedding_Api(); + $store = new Embedding_Store(); + $stats = $store->get_stats(); + ?> +
+

+ + is_configured() ) : ?> +
+

+ Settings → AI and enable the Semantic Search experiment, then fill in your provider, model, and API key.', 'ai' ), + array( 'a' => array( 'href' => array() ) ) + ), + esc_url( admin_url( 'options-general.php?page=ai-wp-admin' ) ) + ); + ?> +

+
+ + +

+ +

+ +
+ + + +
+ +

+ + +
+ + + api = new Embedding_Api(); + $this->store = new Embedding_Store(); + } + + /** + * Generates and persists an embedding for a single post. + * + * Builds the post text from title + stripped content, requests an embedding + * from the configured provider, and writes the result to post meta via the + * embedding store. Returns false if the post does not exist or the API call fails. + * + * @since x.x.x + * + * @param int $post_id Post ID to index. + * @return bool True on success, false if the post is missing or the API fails. + */ + public function index_post( int $post_id ): bool { + $post = get_post( $post_id ); + + if ( ! $post ) { + return false; + } + + $text = $this->get_post_text( $post ); + $embedding = $this->api->generate( $text ); + + if ( null === $embedding ) { + return false; + } + + $this->store->save( $post_id, $embedding, $this->api->get_model() ); + + return true; + } + + /** + * Indexes a batch of posts, stopping on the first API error. + * + * Iterates over the supplied post IDs and calls index_post() for each. + * If any call fails, the loop breaks immediately and the error string from + * Embedding_Api::get_last_error() is included in the return value so the + * caller can surface it to the user instead of silently continuing. + * + * @since x.x.x + * + * @param int[] $ids Post IDs to index. + * @return array{success:int, failed:int, error:string} Counts and the first error message, if any. + */ + public function index_batch( array $ids ): array { + $success = 0; + $failed = 0; + $error = ''; + + foreach ( $ids as $id ) { + if ( $this->index_post( $id ) ) { + ++$success; + } else { + ++$failed; + $error = $this->api->get_last_error(); + break; + } + } + + return compact( 'success', 'failed', 'error' ); + } + + /** + * Builds the text string to embed for a post. + * + * Combines the post title and stripped post content, separated by two + * newlines, so that the embedding captures both the title semantics and + * the body content. + * + * @since x.x.x + * + * @param \WP_Post $post Post object to extract text from. + * @return string Concatenated title and stripped content. + */ + private function get_post_text( \WP_Post $post ): string { + return trim( $post->post_title . "\n\n" . wp_strip_all_tags( $post->post_content ) ); + } +} diff --git a/includes/Experiments/Semantic_Search/List_Table_Integration.php b/includes/Experiments/Semantic_Search/List_Table_Integration.php new file mode 100644 index 000000000..5fa75c11f --- /dev/null +++ b/includes/Experiments/Semantic_Search/List_Table_Integration.php @@ -0,0 +1,148 @@ + + + is_main_query() ) { + return; + } + + // phpcs:ignore WordPress.Security.NonceVerification.Recommended + if ( empty( $_GET[ self::PARAM ] ) ) { + return; + } + + $search_term = $query->get( 's' ); + + if ( ! $search_term ) { + return; + } + + $vector_search = new Vector_Search(); + + if ( ! $vector_search->is_available() ) { + return; + } + + $results = $vector_search->search( + $search_term, + array( + 'limit' => 50, + 'post_type' => (array) $query->get( 'post_type' ) ?: array( 'post', 'page' ), + ) + ); + + if ( empty( $results ) ) { + return; + } + + $ids = array_column( $results, 'id' ); + + $query->set( 's', '' ); + $query->set( 'post__in', $ids ); + $query->set( 'orderby', 'post__in' ); + } +} diff --git a/includes/Experiments/Semantic_Search/REST_Controller.php b/includes/Experiments/Semantic_Search/REST_Controller.php new file mode 100644 index 000000000..0ccf61191 --- /dev/null +++ b/includes/Experiments/Semantic_Search/REST_Controller.php @@ -0,0 +1,215 @@ + 'GET', + 'callback' => array( $this, 'handle_search' ), + 'permission_callback' => static fn() => current_user_can( 'edit_posts' ), + 'args' => array( + 'q' => array( + 'required' => true, + 'type' => 'string', + 'sanitize_callback' => 'sanitize_text_field', + ), + ), + ) + ); + + register_rest_route( + 'ai/v1', + '/semantic-search/index', + array( + 'methods' => 'POST', + 'callback' => array( $this, 'handle_index_batch' ), + 'permission_callback' => static fn() => current_user_can( 'manage_options' ), + ) + ); + + register_rest_route( + 'ai/v1', + '/semantic-search/index/status', + array( + 'methods' => 'GET', + 'callback' => array( $this, 'handle_index_status' ), + 'permission_callback' => static fn() => current_user_can( 'manage_options' ), + ) + ); + + register_rest_route( + 'ai/v1', + '/semantic-search/index/test', + array( + 'methods' => 'GET', + 'callback' => array( $this, 'handle_test_connection' ), + 'permission_callback' => static fn() => current_user_can( 'manage_options' ), + ) + ); + } + + /** + * Handles GET /ai/v1/semantic-search. + * + * Runs a semantic search for the `q` parameter and returns a ranked list of + * posts. Returns `available: false` when the embedding provider is not + * configured so the command palette JS can gracefully suppress the loader. + * + * @since x.x.x + * + * @param \WP_REST_Request $request The incoming REST request. + * @return \WP_REST_Response Response containing `available` flag and `results` array. + */ + public function handle_search( \WP_REST_Request $request ): \WP_REST_Response { + $query = $request->get_param( 'q' ); + $search = new Vector_Search(); + + if ( ! $search->is_available() ) { + return new \WP_REST_Response( array( 'available' => false, 'results' => array() ), 200 ); + } + + $results = $search->search( $query, array( 'limit' => 10 ) ); + + return new \WP_REST_Response( + array( + 'available' => true, + 'results' => $results, + ), + 200 + ); + } + + /** + * Handles POST /ai/v1/semantic-search/index. + * + * Retrieves the next batch of up to 5 unindexed posts and generates their + * embeddings. Returns progress counters (`indexed`, `total`) and a `done` + * flag so the JS loop on the Index_Page knows when to stop. If all attempts + * in the batch fail and an error string is present, `done` is forced to true + * to prevent the client from retrying with a broken API key or network. + * + * @since x.x.x + * + * @return \WP_REST_Response Response containing success/failed counts, progress stats, and done flag. + */ + public function handle_index_batch(): \WP_REST_Response { + $store = new Embedding_Store(); + $indexer = new Indexer(); + + $ids = $store->get_unindexed_ids( array( 'post', 'page' ), 5 ); + $result = $indexer->index_batch( $ids ); + $stats = $store->get_stats(); + + $payload = array_merge( + $result, + array( + 'indexed' => $stats['indexed'], + 'total' => $stats['total'], + 'done' => empty( $store->get_unindexed_ids( array( 'post', 'page' ), 1 ) ), + ) + ); + + if ( 0 === $result['success'] && '' !== $result['error'] ) { + $payload['done'] = true; + } + + return new \WP_REST_Response( $payload, 200 ); + } + + /** + * Handles GET /ai/v1/semantic-search/index/status. + * + * Returns the current indexed/total post counts without triggering any + * indexing work. + * + * @since x.x.x + * + * @return \WP_REST_Response Response containing `total` and `indexed` integer counts. + */ + public function handle_index_status(): \WP_REST_Response { + return new \WP_REST_Response( ( new Embedding_Store() )->get_stats(), 200 ); + } + + /** + * Handles GET /ai/v1/semantic-search/index/test. + * + * Generates an embedding for the string "test" using the current provider + * configuration and returns the model name and vector dimensions on success. + * On failure, returns the exact error string. For Google 404 errors caused by + * an incorrect model name, also returns the list of embedding-capable models + * available for the configured API key so the user knows what to type. + * + * @since x.x.x + * + * @return \WP_REST_Response Response containing `ok`, and either `model`/`dimensions` or `error`. + */ + public function handle_test_connection(): \WP_REST_Response { + $api = new Embedding_Api(); + $embedding = $api->generate( 'test' ); + + if ( null !== $embedding ) { + return new \WP_REST_Response( + array( + 'ok' => true, + 'model' => $api->get_model(), + 'dimensions' => count( $embedding ), + ), + 200 + ); + } + + $payload = array( + 'ok' => false, + 'error' => $api->get_last_error(), + ); + + if ( 'google' === $api->get_provider() ) { + $available = $api->list_google_embedding_models(); + if ( ! empty( $available ) ) { + $payload['available_models'] = $available; + } + } + + return new \WP_REST_Response( $payload, 200 ); + } +} diff --git a/includes/Experiments/Semantic_Search/Semantic_Search.php b/includes/Experiments/Semantic_Search/Semantic_Search.php new file mode 100644 index 000000000..f2963fd91 --- /dev/null +++ b/includes/Experiments/Semantic_Search/Semantic_Search.php @@ -0,0 +1,270 @@ + __( 'Semantic Search', 'ai' ), + 'description' => __( 'Adds AI-powered semantic search to the posts list and command palette. Uses embedding models to find conceptually related content even when keywords don\'t match.', 'ai' ), + 'category' => Experiment_Category::ADMIN, + 'capability' => 'embedding_generation', + ); + } + + /** + * Registers all WordPress hooks for the experiment. + * + * Called by the Features Loader once the experiment is confirmed enabled. + * Initialises the posts list integration and indexing admin page, and adds + * actions for the command palette JS, REST routes, and save_post reindexing. + * + * @since x.x.x + * + * @return void + */ + public function register(): void { + ( new List_Table_Integration() )->register(); + ( new Index_Page() )->init(); + + add_action( 'enqueue_block_editor_assets', array( $this, 'enqueue_command_palette' ) ); + add_action( 'rest_api_init', array( $this, 'register_rest_routes' ) ); + add_action( 'save_post', array( $this, 'on_save_post' ), 10, 2 ); + } + + /** + * Registers each settings field as a WordPress option with REST API exposure. + * + * Called by Settings_Registration::register_settings() for every registered + * feature. Each option is registered with show_in_rest: true so the React + * settings page can read and write values via the WordPress settings REST API + * (/wp/v2/settings). Without this flag, saves from the UI are silently dropped. + * + * @since x.x.x + * + * @return void + */ + public function register_settings(): void { + foreach ( $this->get_settings_fields() as $field ) { + $option_name = static::get_field_option_name( $field['id'] ); + register_setting( + Settings_Registration::OPTION_GROUP, + $option_name, + array( + 'type' => 'string', + 'sanitize_callback' => 'sanitize_text_field', + 'default' => $field['default'] ?? '', + 'show_in_rest' => true, + ) + ); + } + } + + /** + * Returns the field definitions rendered by the React settings DataForm. + * + * IDs use short names (e.g. 'provider'). Abstract_Feature::get_settings_fields_metadata() + * expands them to full option names (e.g. 'wpai_feature_semantic-search_field_provider') + * before passing them to the settings page script module. + * + * @since x.x.x + * + * @return array, + * }> Field definitions for the settings DataForm. + */ + public function get_settings_fields(): array { + return array( + array( + 'id' => 'provider', + 'label' => __( 'Provider', 'ai' ), + 'type' => 'select', + 'default' => 'openai', + 'elements' => array( + array( 'value' => 'openai', 'label' => 'OpenAI' ), + array( 'value' => 'jina', 'label' => 'Jina AI' ), + array( 'value' => 'ollama', 'label' => 'Ollama (local)' ), + array( 'value' => 'google', 'label' => 'Google Gemini' ), + ), + ), + array( + 'id' => 'base_url', + 'label' => __( 'Endpoint URL', 'ai' ), + 'type' => 'text', + 'default' => 'https://api.openai.com/v1/embeddings', + ), + array( + 'id' => 'model', + 'label' => __( 'Model', 'ai' ), + 'type' => 'text', + 'default' => 'text-embedding-3-small', + ), + array( + 'id' => 'score_threshold', + 'label' => __( 'Score Threshold', 'ai' ), + 'type' => 'text', + 'default' => '', + ), + array( + 'id' => 'api_key', + 'label' => __( 'API Key', 'ai' ), + 'type' => 'text', + 'default' => '', + ), + ); + } + + /** + * Enqueues the command palette JS in the block editor. + * + * Reads the asset manifest from build-scripts/experiments/semantic-search.asset.php + * for dependency and version data. Falls back to an empty dependency list if + * the manifest is missing. The script is only enqueued when the embedding + * provider is configured, to avoid registering a no-op command loader. + * + * @since x.x.x + * + * @return void + */ + public function enqueue_command_palette(): void { + $api = new Embedding_Api(); + + if ( ! $api->is_configured() ) { + return; + } + + $handle = 'wpai-semantic-search-command-palette'; + $asset_file = WPAI_PLUGIN_DIR . 'build-scripts/experiments/semantic-search.asset.php'; + $asset = file_exists( $asset_file ) ? require $asset_file : array( 'dependencies' => array(), 'version' => WPAI_VERSION ); // phpcs:ignore WordPressVIPMinimum.Files.IncludingFile.UsingVariable + + wp_enqueue_script( + $handle, + WPAI_PLUGIN_URL . 'build-scripts/experiments/semantic-search.js', + $asset['dependencies'], + $asset['version'], + array( 'in_footer' => true ) + ); + + wp_localize_script( + $handle, + 'wpaiSemanticSearch', + array( + 'restUrl' => rest_url( 'ai/v1/semantic-search' ), + 'nonce' => wp_create_nonce( 'wp_rest' ), + ) + ); + } + + /** + * Delegates REST route registration to REST_Controller. + * + * Hooked to rest_api_init so routes are only registered when the REST API + * is initialised, not on every admin page load. + * + * @since x.x.x + * + * @return void + */ + public function register_rest_routes(): void { + ( new REST_Controller() )->register_routes(); + } + + /** + * Re-indexes a post immediately when it is published or updated. + * + * Skips autosaves, revisions, and non-published posts. Runs synchronously + * which is acceptable for single-post saves; bulk indexing goes through + * POST /ai/v1/semantic-search/index in batches of 5. Does nothing when the + * embedding provider is not configured. + * + * @since x.x.x + * + * @param int $post_id Post ID being saved. + * @param \WP_Post $post Post object being saved. + * @return void + */ + public function on_save_post( int $post_id, \WP_Post $post ): void { + if ( wp_is_post_autosave( $post_id ) || wp_is_post_revision( $post_id ) ) { + return; + } + + if ( 'publish' !== $post->post_status ) { + return; + } + + $api = new Embedding_Api(); + + if ( ! $api->is_configured() ) { + return; + } + + ( new Indexer() )->index_post( $post_id ); + } +} diff --git a/includes/Experiments/Semantic_Search/Vector_Search.php b/includes/Experiments/Semantic_Search/Vector_Search.php new file mode 100644 index 000000000..339eb4959 --- /dev/null +++ b/includes/Experiments/Semantic_Search/Vector_Search.php @@ -0,0 +1,177 @@ +api = new Embedding_Api(); + $this->store = new Embedding_Store(); + } + + /** + * Returns whether the embedding provider is configured and ready to search. + * + * Delegates to Embedding_Api::is_configured(). When this returns false, + * callers should fall back to default WordPress search rather than calling + * search() and receiving an empty result. + * + * @since x.x.x + * + * @return bool True when the provider has the minimum required credentials. + */ + public function is_available(): bool { + return $this->api->is_configured(); + } + + /** + * Returns posts ranked by cosine similarity to the query string. + * + * Generates an embedding for $query, then scores every published post that + * has a stored embedding. Posts scoring below the provider threshold are + * excluded. The remaining results are sorted by score descending and the top + * $args['limit'] entries are returned. + * + * @since x.x.x + * + * @param string $query Search query string. + * @param array{limit?:int, post_type?:string[]} $args { + * @type int $limit Maximum results to return. Default 10. + * @type string[] $post_type Post types to search. Default ['post', 'page']. + * } + * @return array + * Ranked result entries, or an empty array if the query embedding failed. + */ + public function search( string $query, array $args = array() ): array { + $query_embedding = $this->api->generate( $query ); + + if ( null === $query_embedding ) { + return array(); + } + + $limit = $args['limit'] ?? 10; + $post_types = $args['post_type'] ?? array( 'post', 'page' ); + $score_threshold = $this->api->get_score_threshold(); + + $wq = new \WP_Query( + array( + 'post_type' => $post_types, + 'post_status' => 'publish', + 'posts_per_page' => -1, + 'no_found_rows' => true, + 'fields' => 'ids', + ) + ); + + $results = array(); + + foreach ( $wq->posts as $post_id ) { + $embedding = $this->store->get( (int) $post_id ); + + if ( null === $embedding ) { + continue; + } + + $score = $this->cosine_similarity( $query_embedding, $embedding ); + + if ( $score < $score_threshold ) { + continue; + } + + $post = get_post( $post_id ); + + $results[] = array( + 'id' => (int) $post_id, + 'title' => $post->post_title, + 'type' => $post->post_type, + 'url' => (string) ( get_edit_post_link( $post_id, '' ) ?: '#' ), + 'excerpt' => wp_trim_words( wp_strip_all_tags( $post->post_content ), 20 ), + 'score' => round( $score, 4 ), + ); + } + + usort( $results, static fn( $a, $b ) => $b['score'] <=> $a['score'] ); + + return array_slice( $results, 0, $limit ); + } + + /** + * Computes the cosine similarity between two float vectors. + * + * Returns a value in the range [-1, 1] where 1 means the vectors point in + * the same direction (identical semantics) and -1 means opposite directions. + * Returns 0.0 when either vector has zero magnitude to avoid division by zero. + * + * If the two vectors differ in length, only the shorter length is used so + * that mismatched dimensions (e.g. from a model change) don't cause errors. + * + * @since x.x.x + * + * @param float[] $a First embedding vector. + * @param float[] $b Second embedding vector. + * @return float Cosine similarity score in [-1, 1]. + */ + private function cosine_similarity( array $a, array $b ): float { + $dot = 0.0; + $norm_a = 0.0; + $norm_b = 0.0; + $len = min( count( $a ), count( $b ) ); + + for ( $i = 0; $i < $len; $i++ ) { + $dot += $a[ $i ] * $b[ $i ]; + $norm_a += $a[ $i ] ** 2; + $norm_b += $b[ $i ] ** 2; + } + + if ( 0.0 === $norm_a || 0.0 === $norm_b ) { + return 0.0; + } + + return $dot / ( sqrt( $norm_a ) * sqrt( $norm_b ) ); + } +}