diff --git a/modules/api.json b/modules/api.json index 3b25484..2fbef2c 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3517,11 +3517,10 @@ { "name": "q", "in": "query", - "required": true, "schema": { "type": "string" }, - "description": "Search query string" + "description": "Search query string. Required for the keyword/field search; omit it when using the vector-similarity search (the `vector` + `similar` parameters)." }, { "name": "app", @@ -3556,9 +3555,65 @@ "default": 0 }, "description": "Zero-based index of the first result to return (for paging)" + }, + { + "name": "field", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Restrict the query to a single named field (a `field` value from GET /api/search/fields), instead of the default shared site-content/site-title query. Subject to the same field-level security: a field the caller may not see returns 403." + }, + { + "name": "scope", + "in": "query", + "schema": { + "type": "array", + "items": { "type": "string" } + }, + "description": "Collection path(s) to search under, recursive (defaults to the sitewide /db/apps). Same scope semantics as GET /api/search/fields; may be repeated to search several collections." + }, + { + "name": "facet", + "in": "query", + "schema": { + "type": "array", + "items": { "type": "string" } + }, + "description": "Facet drill-down filter, ':' (e.g. site-app:docs). Repeatable: the same dimension repeated combines with OR, different dimensions with AND. ES post_filter semantics — selecting a value narrows the returned results but the facet bucket counts stay stable (they reflect the base query). The app/section parameters are shortcuts for facet=site-app:… / facet=site-section:…" + }, + { + "name": "vector", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Switch to vector-similarity (kNN) search over this named vector field (a `field` value from GET /api/search/fields whose `kind` is `vector`). Requires `similar`; `q` is ignored. The field's embedding model is resolved server-side from its ft:fields record, so the client sends only the field and the text. Subject to the same field-level security as `field`." + }, + { + "name": "similar", + "in": "query", + "schema": { + "type": "string" + }, + "description": "The query text for vector search: it is embedded with the vector field's configured model and matched by similarity. Used only with `vector`." + }, + { + "name": "k", + "in": "query", + "schema": { + "type": "integer", + "default": 10, + "minimum": 1, + "maximum": 100 + }, + "description": "Number of nearest-neighbor results to return for vector search (default 10, clamped to 1–100). The response's `total` equals the number of results returned. Used only with `vector`." } ], "responses": { + "403": { + "description": "The requested field is not available to the caller (field-level security)." + }, "200": { "description": "Search results with total count, relevance scores, and KWIC-highlighted snippets", "content": { @@ -3673,6 +3728,140 @@ } } }, + "/api/search/fields": { + "get": { + "summary": "Discover searchable fields", + "operationId": "fields:list", + "description": "Lists the searchable fields and facets configured under a collection scope, with each field's contract (kind, indexed element(s), analyzer, type, returnable). The result is filtered by a field-level-security policy keyed off the caller's identity: public site-* fields are visible to everyone (including guests); other fields require authentication. Lets a client discover what it can search (and how) before issuing a query to /api/search.", + "tags": [ + "Search" + ], + "parameters": [ + { + "name": "scope", + "in": "query", + "schema": { + "type": "string", + "default": "/db/apps" + }, + "description": "Collection path to introspect (recursive). Defaults to /db/apps (site-wide)." + }, + { + "name": "field", + "in": "query", + "schema": { + "type": "string" + }, + "description": "If given, return only this field's contract." + } + ], + "responses": { + "200": { + "description": "The fields/facets visible to the caller under the scope", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "scope": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The collection path(s) introspected" + }, + "user": { + "type": "string", + "description": "The caller the policy was evaluated for" + }, + "total": { + "type": "integer", + "description": "Number of fields/facets visible to the caller" + }, + "fields": { + "type": "array", + "description": "One record per field/facet, ordered by kind then name", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "Field name or facet dimension" + }, + "kind": { + "type": "string", + "enum": [ + "field", + "facet", + "vector" + ], + "description": "Whether it is a text field, a facet dimension, or a vector field" + }, + "elements": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The element(s) the field is indexed on, across the scope" + }, + "analyzer": { + "description": "Effective analyzer class (text fields only): a single string, or an array when the field is indexed with more than one analyzer across elements" + }, + "type": { + "type": "string", + "description": "Declared XDM type (text fields only), e.g. xs:string" + }, + "returnable": { + "type": "boolean", + "description": "Whether the stored value can be returned (text fields only)" + } + } + } + } + }, + "example": { + "scope": [ + "/db/apps" + ], + "user": "guest", + "total": 3, + "fields": [ + { + "field": "site-app", + "kind": "facet", + "elements": [ + "page" + ] + }, + { + "field": "site-content", + "kind": "field", + "elements": [ + "page" + ], + "analyzer": "org.apache.lucene.analysis.standard.StandardAnalyzer", + "type": "xs:string", + "returnable": true + }, + { + "field": "site-title", + "kind": "field", + "elements": [ + "page" + ], + "analyzer": "org.apache.lucene.analysis.standard.StandardAnalyzer", + "type": "xs:string", + "returnable": true + } + ] + } + } + } + } + } + } + } + }, "/api/site/apps": { "get": { "summary": "List installed apps", diff --git a/modules/api.xq b/modules/api.xq index cc0e07e..3314bcd 100644 --- a/modules/api.xq +++ b/modules/api.xq @@ -26,6 +26,7 @@ import module namespace db="http://exist-db.org/api/db" at "db.xqm"; import module namespace users="http://exist-db.org/api/users" at "users.xqm"; import module namespace packages="http://exist-db.org/api/packages" at "packages.xqm"; import module namespace search="http://exist-db.org/api/search" at "search.xqm"; +import module namespace fields="http://exist-db.org/api/search/fields" at "fields.xqm"; import module namespace site="http://exist-db.org/api/site" at "site.xqm"; (:~ diff --git a/modules/field-policy.xqm b/modules/field-policy.xqm new file mode 100644 index 0000000..423ad04 --- /dev/null +++ b/modules/field-policy.xqm @@ -0,0 +1,41 @@ +(: + : SPDX LGPL-2.1-or-later + : Copyright (C) 2026 The eXist-db Authors + :) +xquery version "3.1"; + +(:~ + : Field-level-security policy — the SINGLE source of truth for "who may see / + : query which search field". Deliberately has NO dependency on ft:fields (or any + : optional core function): it must be importable by /api/search, which has to + : compile on a stock eXist that lacks the ft:fields function (eXist-db/exist#6459). + : The discovery endpoint (fields.xqm, which does use ft:fields) and the search + : endpoint (search.xqm) both import this module so they enforce one policy. + : + : - $fpol:public : visible to everyone, including the unauthenticated guest. + : - $fpol:restricted : field -> the group(s) (any one grants) that may see it; + : a dba always may. + : - any field that is neither public nor restricted is visible to any + : AUTHENTICATED (non-guest) caller. + : There are no per-field ACLs in the index; tune here as new fields/consumers + : appear. + :) +module namespace fpol = "http://exist-db.org/api/search/field-policy"; + +declare variable $fpol:public as xs:string+ := + ("site-content", "site-title", "site-url", "site-app", "site-section"); + +declare variable $fpol:restricted as map(*) := + map { (: "internal-notes": ("editors", "dba") :) }; + +(:~ May a caller with these groups (and dba flag) see/query $field? :) +declare function fpol:visible( + $field as xs:string, $groups as xs:string*, $is-dba as xs:boolean +) as xs:boolean { + if ($is-dba) then true() + else if (map:contains($fpol:restricted, $field)) + then (some $g in $groups satisfies $g = $fpol:restricted($field)) + else if ($field = $fpol:public) then true() + else (: neither public nor restricted -> any authenticated (non-guest) caller :) + exists($groups[. ne "guest"]) +}; diff --git a/modules/fields.xqm b/modules/fields.xqm new file mode 100644 index 0000000..95042cd --- /dev/null +++ b/modules/fields.xqm @@ -0,0 +1,123 @@ +(: + : SPDX LGPL-2.1-or-later + : Copyright (C) 2026 The eXist-db Authors + :) +xquery version "3.1"; + +(:~ + : Sitewide search — field discovery (Phase 2). + : + : Answers "what can I search here, and what is each field's contract?" for a + : consumer (e.g. the Oxygen plugin's field picker) before it issues a query. + : + : Two layers, deliberately separate (the Elasticsearch model — see the + : broaden-/api/search design): + : 1. CATALOG — the full set of configured fields/facets under a scope, from the + : native ft:fields($scope). It reads the resolved Lucene index config via the + : broker, is permission-AGNOSTIC, and is callable by any user (it does NOT + : require the caller to read the admin-only /db/system/config). + : 2. FLS — a group->fields policy decides which catalog entries THIS caller may + : see, applied after the (permission-agnostic) catalog read. Field access + : lives in the policy, never as an ACL on the field. Document-level security + : is already enforced underneath by ft:query-scope/ft:search-scope node + : materialization; this is the field-level layer on top. + :) +module namespace fields = "http://exist-db.org/api/search/fields"; + +(: FLS policy lives in its own module (no ft:fields dependency) so /api/search can + : enforce the same who-sees-what without transitively pulling ft:fields. :) +import module namespace fpol = "http://exist-db.org/api/search/field-policy" at "field-policy.xqm"; + +declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization"; + +declare option output:method "json"; +declare option output:media-type "application/json"; + +(:~ Default scope when the caller doesn't specify one. :) +declare variable $fields:default-scope as xs:string := "/db/apps"; + +(:~ + : CATALOG — the full field/facet set configured under $scope, via native + : ft:fields. Returns one map per configured field/facet/vector OCCURRENCE: + : { field, element, kind: "field"|"facet"|"vector", analyzer?, type?, returnable? } + : (analyzer/type/returnable on text fields only). Permission-agnostic, and it + : aggregates across every collection in scope (so ft:fields("/db/apps") unions + : every sub-app's fields) and always sets field + kind (eXist-db/exist#6459, + : d724759). No descendant-walk or field-presence filter needed here. + :) +declare %private function fields:catalog($scope as xs:string*) as map(*)* { + ft:fields($scope) +}; + +(:~ + : Collapse the per-occurrence catalog to one record per (field, kind), keeping + : the distinct elements it is indexed on AND the distinct analyzers used. A + : shared field can be indexed with different analyzers on different elements + : (e.g. site-content uses StandardAnalyzer on most elements but SimpleAnalyzer on + : the docs xqdoc elements); surfacing both as a list reveals that variance rather + : than hiding it behind whichever occurrence happened to come first. + :) +declare %private function fields:dedup($cat as map(*)*) as map(*)* { + let $sep := codepoints-to-string(9) + for $key in distinct-values($cat ! (?field || $sep || ?kind)) + let $g := $cat[(?field || $sep || ?kind) = $key] + let $first := $g[1] + let $analyzers := distinct-values($g ! ?analyzer)[. ne ""] + return map:merge(( + map { + "field": $first?field, + "kind": $first?kind, + "elements": array { distinct-values($g ! ?element) } + }, + if ($first?kind = "field") then map { + "analyzer": (if (count($analyzers) gt 1) then array { $analyzers } else ($analyzers, ())[1]), + "type": $first?type, + "returnable": $first?returnable + } else () + )) +}; + + +(:~ + : Discover the searchable fields under $scope visible to $user. + : @param $scope one or more collection paths (document paths, recursive) + : @param $user the caller identity map (e.g. $request?user): { name, groups, dba } + :) +declare function fields:discover($scope as xs:string*, $user as map(*)?) as map(*) { + let $name := ($user?name, "guest")[1] + let $groups := ($user?groups, "guest") + let $is-dba := ($user?dba, false())[1] + let $catalog := fields:dedup(fields:catalog($scope)) + let $visible := $catalog[fpol:visible(?field, $groups, $is-dba)] + return map { + "scope": array { $scope }, + "user": $name, + "total": count($visible), + "fields": array { + for $e in $visible + order by $e?kind, $e?field + return $e + } + } +}; + +(:~ + : GET /api/search/fields?scope=/db/apps[&field=site-content] + : Lists the searchable fields/facets the caller may see; with ?field, returns just + : that field's contract. + :) +declare function fields:list($request as map(*)) { + let $scope := + if (exists($request?parameters?scope[. ne ""])) + then $request?parameters?scope[. ne ""] + else $fields:default-scope + let $field := $request?parameters?field + let $result := fields:discover($scope, $request?user) + return + if (exists($field) and $field ne "") + then map:merge(( + map:remove($result, "fields"), + map { "fields": array { $result?fields?*[?field = $field] } } + )) + else $result +}; diff --git a/modules/search.xqm b/modules/search.xqm index 2212f78..ecb7f51 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -21,9 +21,15 @@ xquery version "3.1"; module namespace search="http://exist-db.org/api/search"; import module namespace site="http://exist-db.org/api/site" at "site.xqm"; +(: FLS policy only (no ft:fields) — so /api/search compiles on a stock eXist :) +import module namespace fpol="http://exist-db.org/api/search/field-policy" at "field-policy.xqm"; +import module namespace roaster="http://e-editiones.org/roaster"; import module namespace kwic="http://exist-db.org/xquery/kwic"; declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization"; +(: Vector module (the embedding/kNN extension). Static dependency: this build + : targets the vector-capable integration instance, not a stock eXist. :) +declare namespace vector="http://exist-db.org/xquery/vector"; declare option output:method "json"; declare option output:media-type "application/json"; @@ -47,6 +53,16 @@ declare %private function search:escape($q as xs:string) as xs:string { replace($q, '([+\-&|!(){}\[\]\^"~*?:\\/])', '\\$1') }; +(:~ + : Escape a Lucene FIELD NAME for use as a field selector in a query string. A + : field name may itself contain a colon (e.g. the xqdoc:function discovery + : fields); escape the colon and backslash so the parser reads the whole name as + : the field, with the separating colon added by the caller. + :) +declare %private function search:field-selector($field as xs:string) as xs:string { + replace($field, '([:\\])', '\\$1') +}; + (:~ : Build KWIC highlight fragments for a single hit. Each fragment is a : well-formed XML string: a single root with matched term(s) wrapped in @@ -84,6 +100,101 @@ declare %private function search:facet-counts($hits as node()*, $dimension as xs try { ft:facets($hits, $dimension, ()) } catch * { map {} } }; +(:~ + : Vector-similarity branch of /api/search. + : GET /api/search?vector=&similar=&k=[&scope=] + : + : Discovery-driven: the field's embedding model AND indexed element are read from + : its ft:fields record (the `model` and `element` properties, per + : eXist-db/exist#6459), so the client sends only {field, text}. The text is + : embedded with that model and run as a kNN over the field's element; hits come + : back in the same ES-shaped envelope as keyword search, plus `field`/`model`/ + : `max-score`. + : + : Notes: + : - Uses the node-arg form ft:query-vector(collection($scope)//, vec, k), + : which applies a true cross-document top-k with k authoritative server-side; the + : field form ft:query-field-vector is evaluated per-document in a path step + : (ignores k, scales poorly — eXist-core bug, fix tracked separately). + : - ft:score ordering + subsequence is kept for explicit, robust ranking. + :) +declare %private function search:vector-query( + $field as xs:string, $similar as xs:string?, $scope as xs:string+, + $k as xs:integer, $groups as xs:string*, $is-dba as xs:boolean +) { + if (empty($similar) or $similar = "") + then roaster:response(400, "application/json", + map { "error": "Missing required parameter for vector search: similar" }) + (: Field-level security: the same policy /api/search/fields applies. :) + else if (not(fpol:visible($field, $groups, $is-dba))) + then roaster:response(403, "application/json", + map { "error": "Field not available: " || $field }) + else + (: Resolve the field's embedding model from its ft:fields record. Bind $r + explicitly (avoid ?key in a predicate — eXist mis-handles the cardinality + for >1 item, XPTY0004). :) + let $vrec := (for $r in ft:fields($scope) where $r?kind = "vector" and $r?field = $field return $r)[1] + let $model := $vrec?model + let $element := $vrec?element + return + if (empty($vrec)) + then roaster:response(404, "application/json", + map { "error": "Vector field not found in scope: " || $field }) + else if (empty($model) or $model = "") + then roaster:response(400, "application/json", + map { "error": "Field '" || $field || "' has no embedding model; it cannot embed query text (index it with a model, or query with a precomputed vector)" }) + else + let $vec := vector:embed($similar, $model) + (: Use the node-arg form ft:query-vector(nodes, vec, k), which applies a + true cross-document top-k with k authoritative server-side. The field + form collection($scope)/ft:query-field-vector($field, ...) is evaluated + per-document (a 1-doc kNN per node, unioned), so it both ignores k and + scales poorly — a known eXist-core bug, fix tracked separately. We + target the vector field's indexed element (from ft:fields discovery); + ft:query-vector resolves the field from that element's index config. + Caveat: if an element carries >1 vector field this targets the first, + not necessarily $field — fine for one-field-per-element corpora; when + the core ft:query-field-vector fix lands, switch back to the + field-targeted form for precise multi-field selection. :) + let $hits := ft:query-vector(collection($scope)//*[local-name() = $element], $vec, $k) + let $ranked := + for $h in $hits + let $score := ft:score($h) + order by $score descending + return map { "hit": $h, "score": $score, "uri": document-uri(root($h)) } + let $top := subsequence($ranked, 1, $k) + return map { + "query": $similar, + "field": $field, + "model": $model, + (: total = results returned (= k, or fewer if the corpus is + smaller). NB: the kNN's own k currently over-returns a + candidate pool — tracked by the eXist-core fix to + ft:query-field-vector — so count the post-rank/subsequence + slice, not the raw pool. :) + "total": count($top), + "k": $k, + "max-score": ($top[1]?score, 0)[1], + "results": array { + for $m in $top + let $hit := $m?hit + let $doc-uri := $m?uri + let $app := replace($doc-uri, "^/db/apps/([^/]+)/.*$", "$1") + return map { + "uri": $doc-uri, + "path": $doc-uri, + "title": (string($hit/ancestor-or-self::*[title][1]/title)[. ne ""], "(untitled)")[1], + "app": $app, + "url": site:resolve-link($app, replace($doc-uri, "^/db/apps/[^/]+", "")), + "score": $m?score, + "snippet": serialize( + { substring(string-join($hit//text(), " "), 1, 200) }, + map { "method": "xml" }) + } + } + } +}; + (:~ : Sitewide search over the shared `site-content` field. : GET /api/search?q=array:count&app=docs§ion=functions&limit=20&offset=0 @@ -101,43 +212,102 @@ declare function search:query($request as map(*)) { let $q := $request?parameters?q let $app-filter := $request?parameters?app let $section-filter := $request?parameters?section + (: field: restrict the query to one named field (a /api/search/fields field + value). scope: collection path(s) to search under, recursive (defaults to + the sitewide /db/apps). Both optional; omitting them is today's behavior. :) + let $field := $request?parameters?field[. ne ""] + (: roaster hands a repeatable (array-typed) query param back as an XQuery + array(*) when several values are given, or an atomic when one is — unwrap + to a plain sequence either way. :) + let $scope-list := let $raw := $request?parameters?scope + return if ($raw instance of array(*)) then $raw?* else $raw + let $scope := + if (exists($scope-list[. ne ""])) + then $scope-list[. ne ""] + else "/db/apps" + let $user := $request?user + let $groups := ($user?groups, "guest") + let $is-dba := ($user?dba, false())[1] let $limit := ($request?parameters?limit, 20)[1] cast as xs:integer let $offset := ($request?parameters?offset, 0)[1] cast as xs:integer + (: vector: switch to similarity search over a named vector field. similar: the + query text to embed (server resolves the field's model). Mutually exclusive + with the keyword path — when present, q is not required. :) + let $vector-field := $request?parameters?vector[. ne ""] + (: clamp k to [1, 100]: default 10, hard cap 100 (a kNN result count, not paging) :) + let $k := max((1, min((($request?parameters?k, 10)[1] cast as xs:integer, 100)))) return - if (empty($q) or $q = "") + if (exists($vector-field)) + then search:vector-query($vector-field, $request?parameters?similar, $scope, $k, $groups, $is-dba) + else if (empty($q) or $q = "") then map { "error": "Missing required parameter: q" } + else if (exists($field) and not(fpol:visible($field, $groups, $is-dba))) + (: Field-level security: the same policy /api/search/fields applies — a + field the caller may not see must not be queryable from this connection. :) + then roaster:response(403, "application/json", map { "error": "Field not available: " || $field }) else let $escaped := search:escape($q) - (: Field-scoped query string: body + boosted title. Scope to the - shared field so only contributing result-units match. :) + (: Query string. With ?field, restrict to that one field; otherwise the + default shared-field query (body + boosted title). :) let $query-string := - "site-content:(" || $escaped || ") OR site-title:(" || $escaped || ")^" || $search:title-boost - (: Facet drill-down filters (app/section) — narrow without leaving - the shared field; ES "filter context". :) + if (exists($field)) + then search:field-selector($field) || ":(" || $escaped || ")" + else "site-content:(" || $escaped || ") OR site-title:(" || $escaped || ")^" || $search:title-boost + (: Facet filter — ES post_filter semantics: selecting a value narrows the + returned HITS but NOT the bucket counts, so the counts reflect the base + query and stay stable as the user drills. Sources: ?facet=: + (repeatable; same dim -> OR, different dims -> AND) plus the app/section + shortcuts (= site-app / site-section). :) + let $facet-list := let $raw := $request?parameters?facet + return if ($raw instance of array(*)) then $raw?* else $raw + let $facet-pairs := ( + for $f in $facet-list[. ne ""] + let $d := substring-before($f, ":") + let $v := substring-after($f, ":") + where $d ne "" and $v ne "" + return map { "d": $d, "v": $v }, + if (exists($app-filter) and $app-filter ne "") then map { "d": "site-app", "v": $app-filter } else (), + if (exists($section-filter) and $section-filter ne "") then map { "d": "site-section", "v": $section-filter } else () + ) + (: group values by dimension. NB: avoid ?key in predicates/simple-maps + (e.g. $pairs[?d = $x]) — eXist mis-handles the cardinality (fine for + one item, XPTY0004 for several); bind $p and look up explicitly. :) + let $facet-dims := distinct-values(for $p in $facet-pairs return $p?d) let $facet-filter := - map:merge(( - if (exists($app-filter) and $app-filter ne "") then map { "site-app": $app-filter } else (), - if (exists($section-filter) and $section-filter ne "") then map { "site-section": $section-filter } else () - )) - let $options := - map:merge(( - map { - "default-operator": "and", - "filter-rewrite": "yes", - (: load the producer's display fields so ft:field can return them :) - "fields": ("site-title", "site-url") - }, - if (map:size($facet-filter) gt 0) then map { "facets": $facet-filter } else () - )) - (: Match at document-root level (collection(…)/*) — a single-step - axis preserves ft:score for field queries, unlike //*. :) - let $hits := collection("/db/apps")/*[ft:query(., $query-string, $options)] - (: Facet counts (computed while the Lucene context is intact). :) + map:merge( + for $d in $facet-dims + let $vals := distinct-values(for $p in $facet-pairs where $p?d eq $d return $p?v) + return map { $d: $vals } + ) + let $base-options := + map { + "default-operator": "and", + "filter-rewrite": "yes", + (: load the producer's display fields so ft:field can return them :) + "fields": ("site-title", "site-url") + } + (: Base result set (no facet filter) — drives the stable facet counts. + Match at document-root level (collection(…)/*): a single-step axis + preserves ft:score for field queries, unlike //*. Scope is the caller's + ?scope (recursive) or the sitewide default. :) + let $base-hits := collection($scope)/*[ft:query(., $query-string, $base-options)] let $facets := map { - "site-app": search:facet-counts($hits, "site-app"), - "site-section": search:facet-counts($hits, "site-section") + "site-app": search:facet-counts($base-hits, "site-app"), + "site-section": search:facet-counts($base-hits, "site-section") } + (: Post-filter: when a facet is selected, narrow the hits via Lucene + drill-down; otherwise the hits are the base set. The facets-option + query does NOT collect per-match offsets, so its nodes can't drive + ft:highlight-field-matches/KWIC — intersect the drill set with + $base-hits (which carry the match data) so the returned nodes are the + match-bearing ones, narrowed to the facet selection. (Identity + intersection; preserves the post_filter narrowing and the base-query + facet counts.) :) + let $hits := + if (map:size($facet-filter) gt 0) + then $base-hits intersect collection($scope)/*[ft:query(., $query-string, map:put($base-options, "facets", $facet-filter))] + else $base-hits (: Rank by score, dedup per document (highest-scoring hit wins). :) let $ranked := for $hit in $hits diff --git a/src/test/cypress/e2e/search-facet.cy.js b/src/test/cypress/e2e/search-facet.cy.js new file mode 100644 index 0000000..f9712d8 --- /dev/null +++ b/src/test/cypress/e2e/search-facet.cy.js @@ -0,0 +1,89 @@ +const auth = { username: 'admin', password: '' }; + +// Facet drill-down on /api/search (existdb-openapi#55 / oxygen (c)): +// &facet=: (repeatable; same dim -> OR, different -> AND) +// ES post_filter semantics — selecting a value narrows the returned hits but the +// facet bucket COUNTS stay stable (computed on the base query). Uses standard +// Lucene facet drill-down (ft:query facets option), so it runs on any eXist. +// Self-contained: seeds a fixture with several site-app values. + +const APP = 'cypress-facet'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; + +const SETUP = ` +let $xconf := + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "a.xml", tuning guide content), + xmldb:store("${SCOPE}", "b.xml", more tuning content), + xmldb:store("${SCOPE}", "c.xml", tuning blog content), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — facet drill-down (#55 / oxygen c)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('unfiltered search reports facet buckets with counts', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.eq(3); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); + + it('facet=site-app:docs narrows the hits', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs`, auth }).then(r => { + expect(r.body.total).to.eq(2); + }); + }); + + it('post_filter: bucket counts stay stable when a facet is selected', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs`, auth }).then(r => { + // hits narrowed to 2, but the bucket counts still reflect the base query + expect(r.body.total).to.eq(2); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); + + it('multiple values for one dimension combine with OR', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs&facet=site-app:blog`, auth }).then(r => { + expect(r.body.total).to.eq(3); + }); + }); + + it('the app shortcut is equivalent to facet=site-app:…', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&app=blog`, auth }).then(r => { + expect(r.body.total).to.eq(1); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); +}); diff --git a/src/test/cypress/e2e/search-field-scope.cy.js b/src/test/cypress/e2e/search-field-scope.cy.js new file mode 100644 index 0000000..a69d231 --- /dev/null +++ b/src/test/cypress/e2e/search-field-scope.cy.js @@ -0,0 +1,106 @@ +const auth = { username: 'admin', password: '' }; + +// GET /api/search with the field-scoped query params (existdb-openapi#55): +// field — restrict the query to one named field (a /api/search/fields value) +// scope — collection path(s) to search under +// Field-scoped search uses standard ft:query (NOT ft:fields), so it works on a +// stock eXist — these tests run anywhere, no #6455/#6459 needed. The suite is +// self-contained: it indexes a fixture with a public (site-content) and a +// non-public (secret-notes) field and asserts field isolation + field-level +// security (a field the caller may not see returns 403). + +const APP = 'cypress-fieldscope'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; + +const SETUP = ` +let $xconf := + + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "r1.xml", Onevisible body text), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) { + cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + } + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — field-scoped query (#55)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('field=site-content matches body text', () => { + cy.request({ url: `/api/search?q=visible&field=site-content&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('field isolation: a term only in secret-notes does not match site-content', () => { + cy.request({ url: `/api/search?q=buried&field=site-content&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.total).to.eq(0); + }); + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('scope restricts the search to the given collection', () => { + cy.request({ url: `/api/search?q=visible&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + r.body.results.forEach(hit => expect(hit.path).to.contain(SCOPE)); + }); + }); + + it('FLS: a non-public field is not queryable by guest (403)', () => { + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(403); + expect(r.body).to.have.property('error'); + }); + }); + + it('FLS: a public field IS queryable by guest (200)', () => { + cy.request({ url: `/api/search?q=visible&field=site-content&scope=${enc(SCOPE)}`, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(200); + }); + }); + + it('FLS: a dba may query the non-public field', () => { + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('default (no field) is unchanged: stable envelope shape', () => { + cy.request({ url: `/api/search?q=visible&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body).to.include.all.keys('query', 'total', 'offset', 'limit', 'facets', 'results'); + }); + }); +}); diff --git a/src/test/cypress/e2e/search-fields.cy.js b/src/test/cypress/e2e/search-fields.cy.js new file mode 100644 index 0000000..cdc0036 --- /dev/null +++ b/src/test/cypress/e2e/search-fields.cy.js @@ -0,0 +1,114 @@ +const auth = { username: 'admin', password: '' }; + +// /api/search/fields lists the searchable fields/facets under a scope, filtered +// by the field-level-security policy. It is backed by ft:fields, so it requires +// an eXist with that function (eXist-db/exist#6459); on an eXist without it these +// tests fail at the route. The suite is self-contained: it seeds a fixture +// collection with a public site-content field + a non-public field, scopes +// discovery to that collection, and asserts the contract shape. +// +// FLS differentiation by caller identity (guest sees only public fields; +// authenticated callers also see non-public ones) is validated at the handler +// level (fields:discover with guest/auth/dba identities); an HTTP guest-vs-auth +// assertion here additionally depends on whether the route admits unauthenticated +// callers, which is a separate route-security decision. +const APP = 'cypress-fields-test'; +const SCOPE = `/db/apps/${APP}`; +const DATA = `${SCOPE}/data`; +const CONF = `/db/system/config/db/apps/${APP}/data`; + +const SETUP = ` +let $xconf := + + + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:create-collection("/db/system/config/db/apps/${APP}", "data"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:create-collection("/db/apps/${APP}", "data"), + xmldb:store("${DATA}", "r1.xml", Onearray map serialize), + xmldb:reindex("${DATA}"), + "indexed=" || count(collection("${DATA}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("/db/system/config/db/apps/${APP}")) then xmldb:remove("/db/system/config/db/apps/${APP}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) { + cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + } + }); +} + +describe('/api/search/fields', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + describe('GET /api/search/fields', () => { + it('returns the documented envelope: scope, user, total, fields', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + expect(response.status).to.eq(200); + expect(response.body.scope).to.be.an('array').and.to.include(SCOPE); + expect(response.body).to.have.property('user'); + expect(response.body).to.have.property('total').that.is.a('number'); + expect(response.body.fields).to.be.an('array').and.to.have.length(response.body.total); + }); + }); + + it("reports each field's contract (kind, elements, analyzer, type, returnable)", () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + const sc = response.body.fields.find(f => f.field === 'site-content' && f.kind === 'field'); + expect(sc, 'site-content field record').to.exist; + expect(sc.elements).to.be.an('array').and.to.include('rec'); + expect(sc.analyzer).to.exist; // string, or array when indexed with >1 analyzer + expect(sc).to.have.property('type', 'xs:string'); + expect(sc).to.have.property('returnable', true); + const facet = response.body.fields.find(f => f.field === 'site-app' && f.kind === 'facet'); + expect(facet, 'site-app facet record').to.exist; + }); + }); + + it('narrows to one field with the field parameter', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}&field=site-content`, auth }).then(response => { + expect(response.body.fields).to.have.length.greaterThan(0); + response.body.fields.forEach(f => expect(f.field).to.eq('site-content')); + }); + }); + + it('field-level security: a dba caller sees non-public fields', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + const names = response.body.fields.map(f => f.field); + expect(names, 'public field').to.include('site-content'); + expect(names, 'non-public field, visible to a dba').to.include('secret-notes'); + }); + }); + + it('field-level security: an unauthenticated (guest) caller sees only public fields', () => { + // The route admits unauthenticated callers; identity resolves to guest, so + // the policy returns the public site-* fields only. + cy.request({ url: `/api/search/fields?scope=${SCOPE}` }).then(response => { + expect(response.status).to.eq(200); + expect(response.body.user).to.eq('guest'); + const names = response.body.fields.map(f => f.field); + expect(names, 'public field visible to guest').to.include('site-content'); + expect(names, 'non-public field hidden from guest').to.not.include('secret-notes'); + }); + }); + }); +}); diff --git a/src/test/cypress/e2e/search-vector.cy.js b/src/test/cypress/e2e/search-vector.cy.js new file mode 100644 index 0000000..827ce03 --- /dev/null +++ b/src/test/cypress/e2e/search-vector.cy.js @@ -0,0 +1,93 @@ +const auth = { username: 'admin', password: '' }; + +// Vector-similarity search on /api/search (existdb-openapi#62 / oxygen (d)): +// &vector=&similar=&k= +// Discovery-driven: the client sends only the vector field + query text; the +// server resolves the field's embedding model from its ft:fields record +// (eXist-db/exist#6459) and embeds the text before the kNN. Requires a +// vector-capable eXist (the vector extension module + a local embedding model), +// so it runs on the trio/integration instance, not a stock eXist. +// Self-contained: seeds a small corpus with a text-embedding vector field. + +const APP = 'cypress-vector'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; +const MODEL = 'all-MiniLM-L6-v2'; + +const SETUP = ` +let $xconf := + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "speed.xml", Query performance tuningMake your database queries run much faster by optimizing indexes and caching.), + xmldb:store("${SCOPE}", "cooking.xml", Pasta recipesHow to cook delicious Italian pasta with a fresh tomato and basil sauce.), + xmldb:store("${SCOPE}", "weather.xml", Weather patternsUnderstanding seasonal climate changes and rainfall in the tropics.), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/doc) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — vector similarity (#62 / oxygen d)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('ranks by semantic similarity (client sends only field + text)', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('how do I speed up my database queries')}&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + // the performance doc is most similar to the query, despite no shared keywords with "speed up" + expect(r.body.results[0].uri).to.eq(`${SCOPE}/speed.xml`); + expect(r.body.results[0].score).to.be.greaterThan(0); + }); + }); + + it('resolves and echoes the field model server-side (no model sent by client)', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('fast queries')}&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.field).to.eq('site-embedding'); + expect(r.body.model).to.eq(MODEL); + expect(r.body['max-score']).to.eq(r.body.results[0].score); + }); + }); + + it('k limits the number of results', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('fast queries')}&scope=${enc(SCOPE)}&k=2`, auth }).then(r => { + expect(r.body.k).to.eq(2); + expect(r.body.results).to.have.length(2); + }); + }); + + it('missing similar -> 400', () => { + cy.request({ url: `/api/search?vector=site-embedding&scope=${enc(SCOPE)}`, auth, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(400); + }); + }); + + it('unknown vector field -> 404', () => { + cy.request({ url: `/api/search?vector=no-such-field&similar=${enc('anything')}&scope=${enc(SCOPE)}`, auth, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(404); + }); + }); +});