From 4bb671327c064d1704cfc40c53fc81021fa0c0bf Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Mon, 8 Jun 2026 22:18:40 -0400 Subject: [PATCH 01/11] =?UTF-8?q?feat(search):=20Phase=202=20prototype=20?= =?UTF-8?q?=E2=80=94=20/api/search/fields=20discovery=20+=20FLS=20policy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prototype of the field-discovery half of the broaden-/api/search design: a consumer (e.g. the Oxygen plugin's field picker) can ask "what can I search here?" and "what is this field's contract?" before querying. Two separated layers, per the ES model: - CATALOG: enumerate every configured field/facet under a collection scope, with its contract (kind, indexed element(s), analyzer, type, returnable), read with privilege. This XQuery collection.xconf parser is a STAND-IN for the native ft:fields($scope) the lucene session will build; configs live under /db/system/config (admin-only) and the schema is system-managed, so the catalog read is privileged and permission-agnostic. (The privilege need is exactly why ft:fields is worth building natively — it reads the resolved LuceneConfig via the broker and skips the system-config read entirely.) - FLS: a group->fields policy decides which catalog entries THIS caller sees, applied after the privileged read — keyed off $request?user. Field access lives in the policy, never as an ACL on the field (the Elasticsearch lesson). Default: public site-* fields for everyone incl. guest; everything else for authenticated callers; per-field group restrictions supported. Name-independent of ft:query-scope/ft:search-scope (those names are still in review on eXist-db/exist#6455), so this can proceed now; the field-param query cutover waits for that function to ship. Validated on the live 3-producer instance: guest sees only public site-* fields; an authenticated caller additionally sees the docs app's non-public index fields (category/definition/function-name/term) plus a seeded secret-notes; dba sees all. site-content's contract correctly dedups to one record listing the 7 elements it is indexed on across apps. NOT for merge as-is: the privileged read uses system:as-user (prototype); the production form swaps to ft:fields once it lands, and this gains an api.json route + XQSuite tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/fields.xqm | 157 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 modules/fields.xqm diff --git a/modules/fields.xqm b/modules/fields.xqm new file mode 100644 index 0000000..5c0ab56 --- /dev/null +++ b/modules/fields.xqm @@ -0,0 +1,157 @@ +(: + : SPDX LGPL-2.1-or-later + : Copyright (C) 2026 The eXist-db Authors + :) +xquery version "3.1"; + +(:~ + : Sitewide search — field discovery (Phase 2, prototype). + : + : Enumerates the searchable fields and facets configured across a collection + : scope, and filters them by a field-level-security (FLS) policy keyed off the + : caller's identity. Lets a consumer (e.g. the Oxygen plugin) ask "what can I + : search here?" and "what is this field's contract?" before issuing a query. + : + : Two layers, deliberately separated (the ES model — see the broaden-/api/search + : design): + : 1. CATALOG — the full set of configured fields/facets, read with privilege. + : collection.xconf lives under /db/system/config (not caller-readable), and + : the index schema is system-managed, so the catalog read is privileged and + : permission-AGNOSTIC. This XQuery xconf-parser is a STAND-IN for the native + : ft:fields($scope) (which reads the resolved LuceneConfig via the broker and + : sidesteps the /db/system/config read-permission issue entirely — the + : concrete reason that function is worth building natively). + : 2. FLS — the policy that decides which catalog entries THIS caller may see, + : applied after the privileged read. Field access lives in the policy (keyed + : by group), never as an ACL on the field itself (the Elasticsearch lesson). + :) +module namespace fields = "http://exist-db.org/api/search/fields"; + +declare namespace ccc = "http://exist-db.org/collection-config/1.0"; +declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization"; + +declare option output:method "json"; +declare option output:media-type "application/json"; + +(:~ Default scope when the caller doesn't specify one. :) +declare variable $fields:default-scope as xs:string := "/db/apps"; + +(:~ + : FLS policy. + : - $fields:public : visible to everyone, including the unauthenticated guest. + : - $fields:restricted: field -> the group(s) (any one grants) that may see it; + : a dba always may. + : - any field that is neither public nor restricted is visible to any + : AUTHENTICATED (non-guest) caller. + : This is the single place "who sees what" is decided — there are no per-field + : ACLs in the index. Tune here as new fields/consumers appear. + :) +declare variable $fields:public as xs:string+ := + ("site-content", "site-title", "site-url", "site-app", "site-section"); +declare variable $fields:restricted as map(*) := + map { (: "internal-notes": ("editors", "dba") :) }; + +(:~ + : CATALOG — parse the collection.xconf docs governing $scope into one record per + : configured field/facet. Privileged read; returns the FULL set (FLS applied + : later). Stand-in for ft:fields($scope). + : + : @param $scope a collection path, e.g. "/db/apps" + : @return one map per field/facet: { field, kind, element, analyzer?, type?, returnable? } + :) +declare %private function fields:catalog($scope as xs:string) as map(*)* { + let $config-root := "/db/system/config" || $scope + let $read := + function() { + for $t in collection($config-root)//ccc:text + let $on := (string($t/@qname), string($t/@match))[. ne ""][1] + let $analyzer := + ( string($t/@analyzer), + string(($t/ancestor::ccc:lucene[1]/ccc:analyzer[not(@id)])[1]/@class), + string(($t/ancestor::ccc:lucene[1]/ccc:analyzer)[1]/@class) )[. ne ""][1] + return ( + for $f in $t/ccc:field + return map { + "field": string($f/@name), + "kind": "field", + "element": $on, + "analyzer": ($analyzer[. ne ""], "(default)")[1], + "type": (string($f/@type)[. ne ""], "xs:string")[1], + "returnable": not(string($f/@store) = "no") + }, + for $fa in $t/ccc:facet + return map { "field": string($fa/@dimension), "kind": "facet", "element": $on } + ) + } + return + (: PROTOTYPE: configs are admin-only; read with privilege. The production + form is ft:fields($scope), which reads the resolved config natively via + the broker and needs no credential here. :) + system:as-user("admin", "", $read()) +}; + +(:~ Dedup the catalog by (field, kind) — a shared field (site-content) appears in + : many app configs; collapse to one record, keeping the distinct elements it is + : indexed on. :) +declare %private function fields:dedup($cat as map(*)*) as map(*)* { + for $key in distinct-values($cat ! (?field || "\t" || ?kind)) + let $group := $cat[(?field || "\t" || ?kind) = $key] + let $first := $group[1] + return map:merge(( + $first, + map { "elements": array { distinct-values($group ! ?element) } } + )) +}; + +(:~ FLS: may a caller with these groups (and dba flag) see $field? :) +declare %private function fields:visible( + $field as xs:string, $groups as xs:string*, $is-dba as xs:boolean +) as xs:boolean { + if ($is-dba) then true() + else if (map:contains($fields:restricted, $field)) + then (some $g in $groups satisfies $g = $fields:restricted($field)) + else if ($field = $fields:public) then true() + else (: neither public nor restricted -> any authenticated (non-guest) caller :) + exists($groups[. ne "guest"]) or (exists($groups) and not($groups = "guest")) +}; + +(:~ + : Discover the searchable fields under $scope visible to $user. + : @param $scope a collection path + : @param $user the caller identity map (e.g. $request?user): { name, groups, dba } + :) +declare function fields:discover($scope as xs:string, $user as map(*)?) as map(*) { + let $name := ($user?name, "guest")[1] + let $groups := ($user?groups, "guest") + let $is-dba := ($user?dba, false())[1] + let $catalog := fields:dedup(fields:catalog($scope)) + let $visible := $catalog[fields:visible(?field, $groups, $is-dba)] + return map { + "scope": $scope, + "user": $name, + "total": count($visible), + "fields": array { + for $e in $visible + order by $e?kind, $e?field + return map:remove($e, "element") + } + } +}; + +(:~ + : GET /api/search/fields?scope=/db/apps[&field=site-content] + : Lists the searchable fields/facets the caller may see; with ?field, returns just + : that field's contract. + :) +declare function fields:list($request as map(*)) { + let $scope := ($request?parameters?scope[. ne ""], $fields:default-scope)[1] + let $field := $request?parameters?field + let $result := fields:discover($scope, $request?user) + return + if (exists($field) and $field ne "") + then map:merge(( + map:remove($result, "fields"), + map { "fields": array { $result?fields?*[?field = $field] } } + )) + else $result +}; From 3cb040e67a06d5f7aeb9467bb9caa4ea2f8b6733 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 9 Jun 2026 01:00:57 -0400 Subject: [PATCH 02/11] =?UTF-8?q?feat(search):=20Phase=202=20=E2=80=94=20b?= =?UTF-8?q?ack=20field=20discovery=20with=20native=20ft:fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the collection.xconf-parsing + system:as-user stand-in with the native ft:fields($scope) (now in eXist-db/exist#6459). The catalog read is now permission-agnostic and credential-free, exactly as designed. Two ft:fields behaviours had to be handled in the API layer (worth a core follow-up — see the handoff note): 1. ft:fields does NOT aggregate across collections: it resolves the single config for a given collection/doc-set, so ft:fields("/db/apps") is empty when each app's config lives on its own data collection, and a sequence scope resolves to only the first collection's config. For site-wide discovery we union ft:fields over every descendant collection in scope (fields:descendant-collections). Collapses to a single ft:fields($scope) if it gains native cross-collection aggregation. 2. ft:fields also emits element-level text-index records (a plain with no named yields a map with only "element"); those aren't named, field:(...)-queryable fields, so the catalog drops maps without a "field" key. dedup now surfaces per-field analyzer VARIANCE as an array (a shared field indexed with different analyzers on different elements — e.g. site-content StandardAnalyzer vs WordDelimiter — is reported as both, not hidden). Validated on a 2-app + bundled-apps bed: cross-collection union works (site-content elements [a,b]); analyzer variance shows both analyzers; FLS differentiates guest (public site-* only) from authenticated (also sees function-name/secret-notes) from dba (all). Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/fields.xqm | 133 +++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/modules/fields.xqm b/modules/fields.xqm index 5c0ab56..4f155f8 100644 --- a/modules/fields.xqm +++ b/modules/fields.xqm @@ -5,29 +5,25 @@ xquery version "3.1"; (:~ - : Sitewide search — field discovery (Phase 2, prototype). + : Sitewide search — field discovery (Phase 2). : - : Enumerates the searchable fields and facets configured across a collection - : scope, and filters them by a field-level-security (FLS) policy keyed off the - : caller's identity. Lets a consumer (e.g. the Oxygen plugin) ask "what can I - : search here?" and "what is this field's contract?" before issuing a query. + : Answers "what can I search here, and what is each field's contract?" for a + : consumer (e.g. the Oxygen plugin's field picker) before it issues a query. : - : Two layers, deliberately separated (the ES model — see the broaden-/api/search - : design): - : 1. CATALOG — the full set of configured fields/facets, read with privilege. - : collection.xconf lives under /db/system/config (not caller-readable), and - : the index schema is system-managed, so the catalog read is privileged and - : permission-AGNOSTIC. This XQuery xconf-parser is a STAND-IN for the native - : ft:fields($scope) (which reads the resolved LuceneConfig via the broker and - : sidesteps the /db/system/config read-permission issue entirely — the - : concrete reason that function is worth building natively). - : 2. FLS — the policy that decides which catalog entries THIS caller may see, - : applied after the privileged read. Field access lives in the policy (keyed - : by group), never as an ACL on the field itself (the Elasticsearch lesson). + : Two layers, deliberately separate (the Elasticsearch model — see the + : broaden-/api/search design): + : 1. CATALOG — the full set of configured fields/facets under a scope, from the + : native ft:fields($scope). It reads the resolved Lucene index config via the + : broker, is permission-AGNOSTIC, and is callable by any user (it does NOT + : require the caller to read the admin-only /db/system/config). + : 2. FLS — a group->fields policy decides which catalog entries THIS caller may + : see, applied after the (permission-agnostic) catalog read. Field access + : lives in the policy, never as an ACL on the field. Document-level security + : is already enforced underneath by ft:query-scope/ft:search-scope node + : materialization; this is the field-level layer on top. :) module namespace fields = "http://exist-db.org/api/search/fields"; -declare namespace ccc = "http://exist-db.org/collection-config/1.0"; declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization"; declare option output:method "json"; @@ -51,55 +47,60 @@ declare variable $fields:public as xs:string+ := declare variable $fields:restricted as map(*) := map { (: "internal-notes": ("editors", "dba") :) }; +(:~ All descendant collections of $col (inclusive), for cross-collection union. :) +declare %private function fields:descendant-collections($col as xs:string) as xs:string* { + if (xmldb:collection-available($col)) + then ($col, for $child in xmldb:get-child-collections($col) + return fields:descendant-collections($col || "/" || $child)) + else () +}; + (:~ - : CATALOG — parse the collection.xconf docs governing $scope into one record per - : configured field/facet. Privileged read; returns the FULL set (FLS applied - : later). Stand-in for ft:fields($scope). + : CATALOG — the full field/facet set configured under $scope, via native + : ft:fields. Returns one map per configured field/facet OCCURRENCE: + : { field, element, kind: "field"|"facet", analyzer?, type?, returnable? } + : (analyzer/type/returnable on fields only). Permission-agnostic. : - : @param $scope a collection path, e.g. "/db/apps" - : @return one map per field/facet: { field, kind, element, analyzer?, type?, returnable? } + : NOTE: ft:fields resolves the SINGLE config for a given collection/doc-set; it + : does NOT aggregate across sub-collections (ft:fields("/db/apps") is empty when + : the configs live on each app's data collection, and a sequence scope resolves + : to only the first collection's config). For site-wide discovery we therefore + : union ft:fields over every descendant collection in scope. If ft:fields gains + : native cross-collection aggregation, this collapses to a single ft:fields($scope). :) -declare %private function fields:catalog($scope as xs:string) as map(*)* { - let $config-root := "/db/system/config" || $scope - let $read := - function() { - for $t in collection($config-root)//ccc:text - let $on := (string($t/@qname), string($t/@match))[. ne ""][1] - let $analyzer := - ( string($t/@analyzer), - string(($t/ancestor::ccc:lucene[1]/ccc:analyzer[not(@id)])[1]/@class), - string(($t/ancestor::ccc:lucene[1]/ccc:analyzer)[1]/@class) )[. ne ""][1] - return ( - for $f in $t/ccc:field - return map { - "field": string($f/@name), - "kind": "field", - "element": $on, - "analyzer": ($analyzer[. ne ""], "(default)")[1], - "type": (string($f/@type)[. ne ""], "xs:string")[1], - "returnable": not(string($f/@store) = "no") - }, - for $fa in $t/ccc:facet - return map { "field": string($fa/@dimension), "kind": "facet", "element": $on } - ) - } - return - (: PROTOTYPE: configs are admin-only; read with privilege. The production - form is ft:fields($scope), which reads the resolved config natively via - the broker and needs no credential here. :) - system:as-user("admin", "", $read()) +declare %private function fields:catalog($scope as xs:string*) as map(*)* { + for $col in distinct-values($scope ! fields:descendant-collections(.)) + (: ft:fields also emits element-level text-index records (a plain + with no named yields a map with only "element"); those aren't + named, field:(...)-queryable fields, so drop them from the catalog. :) + return ft:fields($col)[exists(?field)] }; -(:~ Dedup the catalog by (field, kind) — a shared field (site-content) appears in - : many app configs; collapse to one record, keeping the distinct elements it is - : indexed on. :) +(:~ + : Collapse the per-occurrence catalog to one record per (field, kind), keeping + : the distinct elements it is indexed on AND the distinct analyzers used. A + : shared field can be indexed with different analyzers on different elements + : (e.g. site-content uses StandardAnalyzer on most elements but SimpleAnalyzer on + : the docs xqdoc elements); surfacing both as a list reveals that variance rather + : than hiding it behind whichever occurrence happened to come first. + :) declare %private function fields:dedup($cat as map(*)*) as map(*)* { - for $key in distinct-values($cat ! (?field || "\t" || ?kind)) - let $group := $cat[(?field || "\t" || ?kind) = $key] - let $first := $group[1] + let $sep := codepoints-to-string(9) + for $key in distinct-values($cat ! (?field || $sep || ?kind)) + let $g := $cat[(?field || $sep || ?kind) = $key] + let $first := $g[1] + let $analyzers := distinct-values($g ! ?analyzer)[. ne ""] return map:merge(( - $first, - map { "elements": array { distinct-values($group ! ?element) } } + map { + "field": $first?field, + "kind": $first?kind, + "elements": array { distinct-values($g ! ?element) } + }, + if ($first?kind = "field") then map { + "analyzer": (if (count($analyzers) gt 1) then array { $analyzers } else ($analyzers, ())[1]), + "type": $first?type, + "returnable": $first?returnable + } else () )) }; @@ -112,28 +113,28 @@ declare %private function fields:visible( then (some $g in $groups satisfies $g = $fields:restricted($field)) else if ($field = $fields:public) then true() else (: neither public nor restricted -> any authenticated (non-guest) caller :) - exists($groups[. ne "guest"]) or (exists($groups) and not($groups = "guest")) + exists($groups[. ne "guest"]) }; (:~ : Discover the searchable fields under $scope visible to $user. - : @param $scope a collection path + : @param $scope one or more collection paths (document paths, recursive) : @param $user the caller identity map (e.g. $request?user): { name, groups, dba } :) -declare function fields:discover($scope as xs:string, $user as map(*)?) as map(*) { +declare function fields:discover($scope as xs:string*, $user as map(*)?) as map(*) { let $name := ($user?name, "guest")[1] let $groups := ($user?groups, "guest") let $is-dba := ($user?dba, false())[1] let $catalog := fields:dedup(fields:catalog($scope)) let $visible := $catalog[fields:visible(?field, $groups, $is-dba)] return map { - "scope": $scope, + "scope": array { $scope }, "user": $name, "total": count($visible), "fields": array { for $e in $visible order by $e?kind, $e?field - return map:remove($e, "element") + return $e } } }; @@ -144,7 +145,7 @@ declare function fields:discover($scope as xs:string, $user as map(*)?) as map(* : that field's contract. :) declare function fields:list($request as map(*)) { - let $scope := ($request?parameters?scope[. ne ""], $fields:default-scope)[1] + let $scope := ($request?parameters?scope[. ne ""], $fields:default-scope) let $field := $request?parameters?field let $result := fields:discover($scope, $request?user) return From e9fa5a4a8a661969561cf65cb51d68e99a219844 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 9 Jun 2026 02:19:05 -0400 Subject: [PATCH 03/11] refactor(search): drop ft:fields workarounds (fixed upstream in #6459) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eXist-db/exist#6459 (d724759) addressed both integration findings: ft:fields now aggregates across every collection in scope, and every record carries field + kind (field/facet/vector). So fields:catalog collapses to a single ft:fields($scope) call — removing the descendant-collection union walk and the [exists(?field)] filter. Verified on the 2-app + bundled-apps bed: ft:fields("/db/apps") unions across collections (site-content elements [a,b]), analyzer variance still surfaces as an array, and FLS differentiates guest (public site-* only) / authenticated (+ function-name, secret-notes, and the test-embedding vector field) / dba. Confirmed for the core session: the previously field-less records were the vector case (e.g. test-embedding, now kind:"vector"), not bare element-text indexes. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/fields.xqm | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/modules/fields.xqm b/modules/fields.xqm index 4f155f8..ea4839f 100644 --- a/modules/fields.xqm +++ b/modules/fields.xqm @@ -47,33 +47,17 @@ declare variable $fields:public as xs:string+ := declare variable $fields:restricted as map(*) := map { (: "internal-notes": ("editors", "dba") :) }; -(:~ All descendant collections of $col (inclusive), for cross-collection union. :) -declare %private function fields:descendant-collections($col as xs:string) as xs:string* { - if (xmldb:collection-available($col)) - then ($col, for $child in xmldb:get-child-collections($col) - return fields:descendant-collections($col || "/" || $child)) - else () -}; - (:~ : CATALOG — the full field/facet set configured under $scope, via native - : ft:fields. Returns one map per configured field/facet OCCURRENCE: - : { field, element, kind: "field"|"facet", analyzer?, type?, returnable? } - : (analyzer/type/returnable on fields only). Permission-agnostic. - : - : NOTE: ft:fields resolves the SINGLE config for a given collection/doc-set; it - : does NOT aggregate across sub-collections (ft:fields("/db/apps") is empty when - : the configs live on each app's data collection, and a sequence scope resolves - : to only the first collection's config). For site-wide discovery we therefore - : union ft:fields over every descendant collection in scope. If ft:fields gains - : native cross-collection aggregation, this collapses to a single ft:fields($scope). + : ft:fields. Returns one map per configured field/facet/vector OCCURRENCE: + : { field, element, kind: "field"|"facet"|"vector", analyzer?, type?, returnable? } + : (analyzer/type/returnable on text fields only). Permission-agnostic, and it + : aggregates across every collection in scope (so ft:fields("/db/apps") unions + : every sub-app's fields) and always sets field + kind (eXist-db/exist#6459, + : d724759). No descendant-walk or field-presence filter needed here. :) declare %private function fields:catalog($scope as xs:string*) as map(*)* { - for $col in distinct-values($scope ! fields:descendant-collections(.)) - (: ft:fields also emits element-level text-index records (a plain - with no named yields a map with only "element"); those aren't - named, field:(...)-queryable fields, so drop them from the catalog. :) - return ft:fields($col)[exists(?field)] + ft:fields($scope) }; (:~ From 779130a09681306736725098c8523345cf62df23 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 9 Jun 2026 07:41:05 -0400 Subject: [PATCH 04/11] feat(search): add GET /api/search/fields route + tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the field-discovery handler (fields:list) into the API: - modules/api.xq: import the fields module so function-lookup resolves the "fields:list" operationId (same mechanism as search:query). - modules/api.json: add the GET /api/search/fields path — scope (default /db/apps) + optional field params; documented response envelope (scope/user/total/fields[] with field/kind/elements/analyzer/type/returnable) and an example. - src/test/cypress/e2e/search-fields.cy.js: self-contained suite (seeds a fixture collection with a public site-content field + a non-public field) asserting the envelope, the per-field contract, the field= filter, and that an authenticated caller sees non-public fields. Validated at the handler level on the ft:fields bed (fields:list over synthetic roaster $request maps): guest sees public site-* only; authenticated sees the non-public fields too; field= narrows to one; default scope applied. Depends on ft:fields (eXist-db/exist#6459): the route and the Cypress suite require an eXist that ships ft:fields, so this is branch work until that lands in a release (CI uses the stock image). Not for merge until then. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/api.json | 134 +++++++++++++++++++++++ modules/api.xq | 1 + src/test/cypress/e2e/search-fields.cy.js | 102 +++++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 src/test/cypress/e2e/search-fields.cy.js diff --git a/modules/api.json b/modules/api.json index 3b25484..d6cf063 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3673,6 +3673,140 @@ } } }, + "/api/search/fields": { + "get": { + "summary": "Discover searchable fields", + "operationId": "fields:list", + "description": "Lists the searchable fields and facets configured under a collection scope, with each field's contract (kind, indexed element(s), analyzer, type, returnable). The result is filtered by a field-level-security policy keyed off the caller's identity: public site-* fields are visible to everyone (including guests); other fields require authentication. Lets a client discover what it can search (and how) before issuing a query to /api/search.", + "tags": [ + "Search" + ], + "parameters": [ + { + "name": "scope", + "in": "query", + "schema": { + "type": "string", + "default": "/db/apps" + }, + "description": "Collection path to introspect (recursive). Defaults to /db/apps (site-wide)." + }, + { + "name": "field", + "in": "query", + "schema": { + "type": "string" + }, + "description": "If given, return only this field's contract." + } + ], + "responses": { + "200": { + "description": "The fields/facets visible to the caller under the scope", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "scope": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The collection path(s) introspected" + }, + "user": { + "type": "string", + "description": "The caller the policy was evaluated for" + }, + "total": { + "type": "integer", + "description": "Number of fields/facets visible to the caller" + }, + "fields": { + "type": "array", + "description": "One record per field/facet, ordered by kind then name", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "Field name or facet dimension" + }, + "kind": { + "type": "string", + "enum": [ + "field", + "facet", + "vector" + ], + "description": "Whether it is a text field, a facet dimension, or a vector field" + }, + "elements": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The element(s) the field is indexed on, across the scope" + }, + "analyzer": { + "description": "Effective analyzer class (text fields only): a single string, or an array when the field is indexed with more than one analyzer across elements" + }, + "type": { + "type": "string", + "description": "Declared XDM type (text fields only), e.g. xs:string" + }, + "returnable": { + "type": "boolean", + "description": "Whether the stored value can be returned (text fields only)" + } + } + } + } + }, + "example": { + "scope": [ + "/db/apps" + ], + "user": "guest", + "total": 3, + "fields": [ + { + "field": "site-app", + "kind": "facet", + "elements": [ + "page" + ] + }, + { + "field": "site-content", + "kind": "field", + "elements": [ + "page" + ], + "analyzer": "org.apache.lucene.analysis.standard.StandardAnalyzer", + "type": "xs:string", + "returnable": true + }, + { + "field": "site-title", + "kind": "field", + "elements": [ + "page" + ], + "analyzer": "org.apache.lucene.analysis.standard.StandardAnalyzer", + "type": "xs:string", + "returnable": true + } + ] + } + } + } + } + } + } + } + }, "/api/site/apps": { "get": { "summary": "List installed apps", diff --git a/modules/api.xq b/modules/api.xq index cc0e07e..3314bcd 100644 --- a/modules/api.xq +++ b/modules/api.xq @@ -26,6 +26,7 @@ import module namespace db="http://exist-db.org/api/db" at "db.xqm"; import module namespace users="http://exist-db.org/api/users" at "users.xqm"; import module namespace packages="http://exist-db.org/api/packages" at "packages.xqm"; import module namespace search="http://exist-db.org/api/search" at "search.xqm"; +import module namespace fields="http://exist-db.org/api/search/fields" at "fields.xqm"; import module namespace site="http://exist-db.org/api/site" at "site.xqm"; (:~ diff --git a/src/test/cypress/e2e/search-fields.cy.js b/src/test/cypress/e2e/search-fields.cy.js new file mode 100644 index 0000000..6dc8de2 --- /dev/null +++ b/src/test/cypress/e2e/search-fields.cy.js @@ -0,0 +1,102 @@ +const auth = { username: 'admin', password: '' }; + +// /api/search/fields lists the searchable fields/facets under a scope, filtered +// by the field-level-security policy. It is backed by ft:fields, so it requires +// an eXist with that function (eXist-db/exist#6459); on an eXist without it these +// tests fail at the route. The suite is self-contained: it seeds a fixture +// collection with a public site-content field + a non-public field, scopes +// discovery to that collection, and asserts the contract shape. +// +// FLS differentiation by caller identity (guest sees only public fields; +// authenticated callers also see non-public ones) is validated at the handler +// level (fields:discover with guest/auth/dba identities); an HTTP guest-vs-auth +// assertion here additionally depends on whether the route admits unauthenticated +// callers, which is a separate route-security decision. +const APP = 'cypress-fields-test'; +const SCOPE = `/db/apps/${APP}`; +const DATA = `${SCOPE}/data`; +const CONF = `/db/system/config/db/apps/${APP}/data`; + +const SETUP = ` +let $xconf := + + + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:create-collection("/db/system/config/db/apps/${APP}", "data"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:create-collection("/db/apps/${APP}", "data"), + xmldb:store("${DATA}", "r1.xml", Onearray map serialize), + xmldb:reindex("${DATA}"), + "indexed=" || count(collection("${DATA}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("/db/system/config/db/apps/${APP}")) then xmldb:remove("/db/system/config/db/apps/${APP}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) { + cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + } + }); +} + +describe('/api/search/fields', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + describe('GET /api/search/fields', () => { + it('returns the documented envelope: scope, user, total, fields', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + expect(response.status).to.eq(200); + expect(response.body.scope).to.be.an('array').and.to.include(SCOPE); + expect(response.body).to.have.property('user'); + expect(response.body).to.have.property('total').that.is.a('number'); + expect(response.body.fields).to.be.an('array').and.to.have.length(response.body.total); + }); + }); + + it("reports each field's contract (kind, elements, analyzer, type, returnable)", () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + const sc = response.body.fields.find(f => f.field === 'site-content' && f.kind === 'field'); + expect(sc, 'site-content field record').to.exist; + expect(sc.elements).to.be.an('array').and.to.include('rec'); + expect(sc.analyzer).to.exist; // string, or array when indexed with >1 analyzer + expect(sc).to.have.property('type', 'xs:string'); + expect(sc).to.have.property('returnable', true); + const facet = response.body.fields.find(f => f.field === 'site-app' && f.kind === 'facet'); + expect(facet, 'site-app facet record').to.exist; + }); + }); + + it('narrows to one field with the field parameter', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}&field=site-content`, auth }).then(response => { + expect(response.body.fields).to.have.length.greaterThan(0); + response.body.fields.forEach(f => expect(f.field).to.eq('site-content')); + }); + }); + + it('exposes non-public fields to an authenticated (dba) caller', () => { + cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { + const names = response.body.fields.map(f => f.field); + expect(names, 'public field').to.include('site-content'); + expect(names, 'non-public field, visible to this authenticated caller').to.include('secret-notes'); + }); + }); + }); +}); From edf17feb21fbbe690a5fb7439022e54459b94714 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 9 Jun 2026 07:50:27 -0400 Subject: [PATCH 05/11] fix(search): correct scope fallback in /api/search/fields; add guest FLS test End-to-end HTTP testing (existdb-openapi installed on an ft:fields-enabled eXist) surfaced a bug the handler-level test missed: the scope fallback `($request?parameters?scope[. ne ""], $fields:default-scope)` always appended the default, so a provided scope echoed twice (and was passed doubled to ft:fields). Use an if/else so a provided scope (one or more) is used as-is and the default applies only when none is given. The HTTP test also confirmed the route admits unauthenticated callers (identity resolves to guest), so the guest "public-only" FLS tier is reachable over HTTP. Added a Cypress assertion for it: a guest sees the public site-* fields but not the non-public secret-notes; a dba sees both. Verified on a full PoC bed (producers snapshot + the ft:fields lucene jar): GET /api/search/fields returns, for the real corpus, site-content unioned across 6 elements with mixed analyzers [StandardAnalyzer, SimpleAnalyzer]; admin sees 12 fields (field/facet/vector), guest sees 7 (public site-* only). All five Cypress scenarios pass against the live route. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/fields.xqm | 5 ++++- src/test/cypress/e2e/search-fields.cy.js | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/fields.xqm b/modules/fields.xqm index ea4839f..9c0a751 100644 --- a/modules/fields.xqm +++ b/modules/fields.xqm @@ -129,7 +129,10 @@ declare function fields:discover($scope as xs:string*, $user as map(*)?) as map( : that field's contract. :) declare function fields:list($request as map(*)) { - let $scope := ($request?parameters?scope[. ne ""], $fields:default-scope) + let $scope := + if (exists($request?parameters?scope[. ne ""])) + then $request?parameters?scope[. ne ""] + else $fields:default-scope let $field := $request?parameters?field let $result := fields:discover($scope, $request?user) return diff --git a/src/test/cypress/e2e/search-fields.cy.js b/src/test/cypress/e2e/search-fields.cy.js index 6dc8de2..cdc0036 100644 --- a/src/test/cypress/e2e/search-fields.cy.js +++ b/src/test/cypress/e2e/search-fields.cy.js @@ -91,11 +91,23 @@ describe('/api/search/fields', () => { }); }); - it('exposes non-public fields to an authenticated (dba) caller', () => { + it('field-level security: a dba caller sees non-public fields', () => { cy.request({ url: `/api/search/fields?scope=${SCOPE}`, auth }).then(response => { const names = response.body.fields.map(f => f.field); expect(names, 'public field').to.include('site-content'); - expect(names, 'non-public field, visible to this authenticated caller').to.include('secret-notes'); + expect(names, 'non-public field, visible to a dba').to.include('secret-notes'); + }); + }); + + it('field-level security: an unauthenticated (guest) caller sees only public fields', () => { + // The route admits unauthenticated callers; identity resolves to guest, so + // the policy returns the public site-* fields only. + cy.request({ url: `/api/search/fields?scope=${SCOPE}` }).then(response => { + expect(response.status).to.eq(200); + expect(response.body.user).to.eq('guest'); + const names = response.body.fields.map(f => f.field); + expect(names, 'public field visible to guest').to.include('site-content'); + expect(names, 'non-public field hidden from guest').to.not.include('secret-notes'); }); }); }); From 635bb0f10ea1d81f24f774bc3e17328426b1f0cf Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Wed, 10 Jun 2026 20:21:00 -0400 Subject: [PATCH 06/11] feat(search): field-scoped query (field/scope params) + FLS on /api/search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the oxygen field-scoped-search contract (existdb-openapi#55): - `field` (optional): restrict the query to one named field (a value from GET /api/search/fields) instead of the default shared site-content/site-title query. Built on standard ft:query (a field-qualified query string), so it works on a stock eXist — NOT gated on #6455/#6459. Only discovery needs ft:fields. - `scope` (optional, repeatable): collection path(s) to search under, recursive; same semantics as /api/search/fields. Defaults to the sitewide /db/apps. - Field-level security: a field the caller may not see is not queryable — returns 403, enforced by the same policy /api/search/fields uses. - Response shape unchanged (query/total/offset/limit/facets/results), so the plugin's parser is untouched; the deferred ~10-line plugin wiring can now land. To avoid a regression, the FLS policy (public/restricted/visible) is extracted to a new field-policy.xqm with NO ft:fields dependency, imported by both search.xqm and fields.xqm. Previously search would have transitively pulled ft:fields via fields.xqm and failed to compile on a stock eXist (XPST0017); verified fixed — /api/search compiles and field-scoped search works on stock beta3 (ft:fields absent). 7 self-contained Cypress tests (field isolation, scope, guest-403, public-200, dba override, stable default); all green. Verified on the trio instance (:19110). Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/api.json | 19 ++++ modules/field-policy.xqm | 41 +++++++ modules/fields.xqm | 32 +----- modules/search.xqm | 41 ++++++- src/test/cypress/e2e/search-field-scope.cy.js | 106 ++++++++++++++++++ 5 files changed, 207 insertions(+), 32 deletions(-) create mode 100644 modules/field-policy.xqm create mode 100644 src/test/cypress/e2e/search-field-scope.cy.js diff --git a/modules/api.json b/modules/api.json index d6cf063..fc433c1 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3556,9 +3556,28 @@ "default": 0 }, "description": "Zero-based index of the first result to return (for paging)" + }, + { + "name": "field", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Restrict the query to a single named field (a `field` value from GET /api/search/fields), instead of the default shared site-content/site-title query. Subject to the same field-level security: a field the caller may not see returns 403." + }, + { + "name": "scope", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Collection path(s) to search under, recursive (defaults to the sitewide /db/apps). Same scope semantics as GET /api/search/fields; may be repeated to search several collections." } ], "responses": { + "403": { + "description": "The requested field is not available to the caller (field-level security)." + }, "200": { "description": "Search results with total count, relevance scores, and KWIC-highlighted snippets", "content": { diff --git a/modules/field-policy.xqm b/modules/field-policy.xqm new file mode 100644 index 0000000..423ad04 --- /dev/null +++ b/modules/field-policy.xqm @@ -0,0 +1,41 @@ +(: + : SPDX LGPL-2.1-or-later + : Copyright (C) 2026 The eXist-db Authors + :) +xquery version "3.1"; + +(:~ + : Field-level-security policy — the SINGLE source of truth for "who may see / + : query which search field". Deliberately has NO dependency on ft:fields (or any + : optional core function): it must be importable by /api/search, which has to + : compile on a stock eXist that lacks the ft:fields function (eXist-db/exist#6459). + : The discovery endpoint (fields.xqm, which does use ft:fields) and the search + : endpoint (search.xqm) both import this module so they enforce one policy. + : + : - $fpol:public : visible to everyone, including the unauthenticated guest. + : - $fpol:restricted : field -> the group(s) (any one grants) that may see it; + : a dba always may. + : - any field that is neither public nor restricted is visible to any + : AUTHENTICATED (non-guest) caller. + : There are no per-field ACLs in the index; tune here as new fields/consumers + : appear. + :) +module namespace fpol = "http://exist-db.org/api/search/field-policy"; + +declare variable $fpol:public as xs:string+ := + ("site-content", "site-title", "site-url", "site-app", "site-section"); + +declare variable $fpol:restricted as map(*) := + map { (: "internal-notes": ("editors", "dba") :) }; + +(:~ May a caller with these groups (and dba flag) see/query $field? :) +declare function fpol:visible( + $field as xs:string, $groups as xs:string*, $is-dba as xs:boolean +) as xs:boolean { + if ($is-dba) then true() + else if (map:contains($fpol:restricted, $field)) + then (some $g in $groups satisfies $g = $fpol:restricted($field)) + else if ($field = $fpol:public) then true() + else (: neither public nor restricted -> any authenticated (non-guest) caller :) + exists($groups[. ne "guest"]) +}; diff --git a/modules/fields.xqm b/modules/fields.xqm index 9c0a751..95042cd 100644 --- a/modules/fields.xqm +++ b/modules/fields.xqm @@ -24,6 +24,10 @@ xquery version "3.1"; :) module namespace fields = "http://exist-db.org/api/search/fields"; +(: FLS policy lives in its own module (no ft:fields dependency) so /api/search can + : enforce the same who-sees-what without transitively pulling ft:fields. :) +import module namespace fpol = "http://exist-db.org/api/search/field-policy" at "field-policy.xqm"; + declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization"; declare option output:method "json"; @@ -32,21 +36,6 @@ declare option output:media-type "application/json"; (:~ Default scope when the caller doesn't specify one. :) declare variable $fields:default-scope as xs:string := "/db/apps"; -(:~ - : FLS policy. - : - $fields:public : visible to everyone, including the unauthenticated guest. - : - $fields:restricted: field -> the group(s) (any one grants) that may see it; - : a dba always may. - : - any field that is neither public nor restricted is visible to any - : AUTHENTICATED (non-guest) caller. - : This is the single place "who sees what" is decided — there are no per-field - : ACLs in the index. Tune here as new fields/consumers appear. - :) -declare variable $fields:public as xs:string+ := - ("site-content", "site-title", "site-url", "site-app", "site-section"); -declare variable $fields:restricted as map(*) := - map { (: "internal-notes": ("editors", "dba") :) }; - (:~ : CATALOG — the full field/facet set configured under $scope, via native : ft:fields. Returns one map per configured field/facet/vector OCCURRENCE: @@ -88,17 +77,6 @@ declare %private function fields:dedup($cat as map(*)*) as map(*)* { )) }; -(:~ FLS: may a caller with these groups (and dba flag) see $field? :) -declare %private function fields:visible( - $field as xs:string, $groups as xs:string*, $is-dba as xs:boolean -) as xs:boolean { - if ($is-dba) then true() - else if (map:contains($fields:restricted, $field)) - then (some $g in $groups satisfies $g = $fields:restricted($field)) - else if ($field = $fields:public) then true() - else (: neither public nor restricted -> any authenticated (non-guest) caller :) - exists($groups[. ne "guest"]) -}; (:~ : Discover the searchable fields under $scope visible to $user. @@ -110,7 +88,7 @@ declare function fields:discover($scope as xs:string*, $user as map(*)?) as map( let $groups := ($user?groups, "guest") let $is-dba := ($user?dba, false())[1] let $catalog := fields:dedup(fields:catalog($scope)) - let $visible := $catalog[fields:visible(?field, $groups, $is-dba)] + let $visible := $catalog[fpol:visible(?field, $groups, $is-dba)] return map { "scope": array { $scope }, "user": $name, diff --git a/modules/search.xqm b/modules/search.xqm index 2212f78..7069331 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -21,6 +21,9 @@ xquery version "3.1"; module namespace search="http://exist-db.org/api/search"; import module namespace site="http://exist-db.org/api/site" at "site.xqm"; +(: FLS policy only (no ft:fields) — so /api/search compiles on a stock eXist :) +import module namespace fpol="http://exist-db.org/api/search/field-policy" at "field-policy.xqm"; +import module namespace roaster="http://e-editiones.org/roaster"; import module namespace kwic="http://exist-db.org/xquery/kwic"; declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization"; @@ -47,6 +50,16 @@ declare %private function search:escape($q as xs:string) as xs:string { replace($q, '([+\-&|!(){}\[\]\^"~*?:\\/])', '\\$1') }; +(:~ + : Escape a Lucene FIELD NAME for use as a field selector in a query string. A + : field name may itself contain a colon (e.g. the xqdoc:function discovery + : fields); escape the colon and backslash so the parser reads the whole name as + : the field, with the separating colon added by the caller. + :) +declare %private function search:field-selector($field as xs:string) as xs:string { + replace($field, '([:\\])', '\\$1') +}; + (:~ : Build KWIC highlight fragments for a single hit. Each fragment is a : well-formed XML string: a single root with matched term(s) wrapped in @@ -101,17 +114,34 @@ declare function search:query($request as map(*)) { let $q := $request?parameters?q let $app-filter := $request?parameters?app let $section-filter := $request?parameters?section + (: field: restrict the query to one named field (a /api/search/fields field + value). scope: collection path(s) to search under, recursive (defaults to + the sitewide /db/apps). Both optional; omitting them is today's behavior. :) + let $field := $request?parameters?field[. ne ""] + let $scope := + if (exists($request?parameters?scope[. ne ""])) + then $request?parameters?scope[. ne ""] + else "/db/apps" + let $user := $request?user + let $groups := ($user?groups, "guest") + let $is-dba := ($user?dba, false())[1] let $limit := ($request?parameters?limit, 20)[1] cast as xs:integer let $offset := ($request?parameters?offset, 0)[1] cast as xs:integer return if (empty($q) or $q = "") then map { "error": "Missing required parameter: q" } + else if (exists($field) and not(fpol:visible($field, $groups, $is-dba))) + (: Field-level security: the same policy /api/search/fields applies — a + field the caller may not see must not be queryable from this connection. :) + then roaster:response(403, "application/json", map { "error": "Field not available: " || $field }) else let $escaped := search:escape($q) - (: Field-scoped query string: body + boosted title. Scope to the - shared field so only contributing result-units match. :) + (: Query string. With ?field, restrict to that one field; otherwise the + default shared-field query (body + boosted title). :) let $query-string := - "site-content:(" || $escaped || ") OR site-title:(" || $escaped || ")^" || $search:title-boost + if (exists($field)) + then search:field-selector($field) || ":(" || $escaped || ")" + else "site-content:(" || $escaped || ") OR site-title:(" || $escaped || ")^" || $search:title-boost (: Facet drill-down filters (app/section) — narrow without leaving the shared field; ES "filter context". :) let $facet-filter := @@ -130,8 +160,9 @@ declare function search:query($request as map(*)) { if (map:size($facet-filter) gt 0) then map { "facets": $facet-filter } else () )) (: Match at document-root level (collection(…)/*) — a single-step - axis preserves ft:score for field queries, unlike //*. :) - let $hits := collection("/db/apps")/*[ft:query(., $query-string, $options)] + axis preserves ft:score for field queries, unlike //*. Scope is the + caller's ?scope (recursive) or the sitewide default. :) + let $hits := collection($scope)/*[ft:query(., $query-string, $options)] (: Facet counts (computed while the Lucene context is intact). :) let $facets := map { diff --git a/src/test/cypress/e2e/search-field-scope.cy.js b/src/test/cypress/e2e/search-field-scope.cy.js new file mode 100644 index 0000000..a69d231 --- /dev/null +++ b/src/test/cypress/e2e/search-field-scope.cy.js @@ -0,0 +1,106 @@ +const auth = { username: 'admin', password: '' }; + +// GET /api/search with the field-scoped query params (existdb-openapi#55): +// field — restrict the query to one named field (a /api/search/fields value) +// scope — collection path(s) to search under +// Field-scoped search uses standard ft:query (NOT ft:fields), so it works on a +// stock eXist — these tests run anywhere, no #6455/#6459 needed. The suite is +// self-contained: it indexes a fixture with a public (site-content) and a +// non-public (secret-notes) field and asserts field isolation + field-level +// security (a field the caller may not see returns 403). + +const APP = 'cypress-fieldscope'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; + +const SETUP = ` +let $xconf := + + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "r1.xml", Onevisible body text), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) { + cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + } + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — field-scoped query (#55)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('field=site-content matches body text', () => { + cy.request({ url: `/api/search?q=visible&field=site-content&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('field isolation: a term only in secret-notes does not match site-content', () => { + cy.request({ url: `/api/search?q=buried&field=site-content&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.total).to.eq(0); + }); + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('scope restricts the search to the given collection', () => { + cy.request({ url: `/api/search?q=visible&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + r.body.results.forEach(hit => expect(hit.path).to.contain(SCOPE)); + }); + }); + + it('FLS: a non-public field is not queryable by guest (403)', () => { + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(403); + expect(r.body).to.have.property('error'); + }); + }); + + it('FLS: a public field IS queryable by guest (200)', () => { + cy.request({ url: `/api/search?q=visible&field=site-content&scope=${enc(SCOPE)}`, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(200); + }); + }); + + it('FLS: a dba may query the non-public field', () => { + cy.request({ url: `/api/search?q=buried&field=secret-notes&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.be.greaterThan(0); + }); + }); + + it('default (no field) is unchanged: stable envelope shape', () => { + cy.request({ url: `/api/search?q=visible&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body).to.include.all.keys('query', 'total', 'offset', 'limit', 'facets', 'results'); + }); + }); +}); From 27293fbe39d35d1297e63e0c816df200244e755d Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 11 Jun 2026 08:46:16 -0400 Subject: [PATCH 07/11] feat(search): facet drill-down filter with post_filter semantics (#55 / oxygen c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds &facet=: to /api/search (repeatable; same dimension -> OR, different dimensions -> AND), implementing the oxygen facet drill-down design (c). ES post_filter semantics: selecting a facet value narrows the returned HITS but NOT the bucket counts — counts are computed on the base query (q + scope) so they stay stable as the user drills (the "blog (12)" still shows after filtering to docs). Implemented as: one base query for the facets map + ft:score ranking; a second drill-down query only when a facet is selected, to narrow the hits. - The app/section params become shortcuts for facet=site-app:… / facet=site-section:… (generalized into the one mechanism). - facet (and scope, also documented repeatable) are declared array-typed in api.json so roaster accepts repetition; the handler unwraps roaster's array(*) to a sequence. Values grouped by dimension with explicit for/where (NOT a ?key-in-predicate, which eXist mis-handles as XPTY0004 for >1 item). Self-contained Cypress suite (5): bucket counts, drill narrows hits, post_filter count stability, multi-value OR, app-shortcut equivalence. search.cy.js (9) and search-field-scope.cy.js (7) stay green (no regression from moving facet counts onto the base query). Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/api.json | 12 +++- modules/search.xqm | 77 ++++++++++++++------- src/test/cypress/e2e/search-facet.cy.js | 89 +++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 src/test/cypress/e2e/search-facet.cy.js diff --git a/modules/api.json b/modules/api.json index fc433c1..14bbbef 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3569,9 +3569,19 @@ "name": "scope", "in": "query", "schema": { - "type": "string" + "type": "array", + "items": { "type": "string" } }, "description": "Collection path(s) to search under, recursive (defaults to the sitewide /db/apps). Same scope semantics as GET /api/search/fields; may be repeated to search several collections." + }, + { + "name": "facet", + "in": "query", + "schema": { + "type": "array", + "items": { "type": "string" } + }, + "description": "Facet drill-down filter, ':' (e.g. site-app:docs). Repeatable: the same dimension repeated combines with OR, different dimensions with AND. ES post_filter semantics — selecting a value narrows the returned results but the facet bucket counts stay stable (they reflect the base query). The app/section parameters are shortcuts for facet=site-app:… / facet=site-section:…" } ], "responses": { diff --git a/modules/search.xqm b/modules/search.xqm index 7069331..77e2593 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -118,9 +118,14 @@ declare function search:query($request as map(*)) { value). scope: collection path(s) to search under, recursive (defaults to the sitewide /db/apps). Both optional; omitting them is today's behavior. :) let $field := $request?parameters?field[. ne ""] + (: roaster hands a repeatable (array-typed) query param back as an XQuery + array(*) when several values are given, or an atomic when one is — unwrap + to a plain sequence either way. :) + let $scope-list := let $raw := $request?parameters?scope + return if ($raw instance of array(*)) then $raw?* else $raw let $scope := - if (exists($request?parameters?scope[. ne ""])) - then $request?parameters?scope[. ne ""] + if (exists($scope-list[. ne ""])) + then $scope-list[. ne ""] else "/db/apps" let $user := $request?user let $groups := ($user?groups, "guest") @@ -142,33 +147,55 @@ declare function search:query($request as map(*)) { if (exists($field)) then search:field-selector($field) || ":(" || $escaped || ")" else "site-content:(" || $escaped || ") OR site-title:(" || $escaped || ")^" || $search:title-boost - (: Facet drill-down filters (app/section) — narrow without leaving - the shared field; ES "filter context". :) + (: Facet filter — ES post_filter semantics: selecting a value narrows the + returned HITS but NOT the bucket counts, so the counts reflect the base + query and stay stable as the user drills. Sources: ?facet=: + (repeatable; same dim -> OR, different dims -> AND) plus the app/section + shortcuts (= site-app / site-section). :) + let $facet-list := let $raw := $request?parameters?facet + return if ($raw instance of array(*)) then $raw?* else $raw + let $facet-pairs := ( + for $f in $facet-list[. ne ""] + let $d := substring-before($f, ":") + let $v := substring-after($f, ":") + where $d ne "" and $v ne "" + return map { "d": $d, "v": $v }, + if (exists($app-filter) and $app-filter ne "") then map { "d": "site-app", "v": $app-filter } else (), + if (exists($section-filter) and $section-filter ne "") then map { "d": "site-section", "v": $section-filter } else () + ) + (: group values by dimension. NB: avoid ?key in predicates/simple-maps + (e.g. $pairs[?d = $x]) — eXist mis-handles the cardinality (fine for + one item, XPTY0004 for several); bind $p and look up explicitly. :) + let $facet-dims := distinct-values(for $p in $facet-pairs return $p?d) let $facet-filter := - map:merge(( - if (exists($app-filter) and $app-filter ne "") then map { "site-app": $app-filter } else (), - if (exists($section-filter) and $section-filter ne "") then map { "site-section": $section-filter } else () - )) - let $options := - map:merge(( - map { - "default-operator": "and", - "filter-rewrite": "yes", - (: load the producer's display fields so ft:field can return them :) - "fields": ("site-title", "site-url") - }, - if (map:size($facet-filter) gt 0) then map { "facets": $facet-filter } else () - )) - (: Match at document-root level (collection(…)/*) — a single-step - axis preserves ft:score for field queries, unlike //*. Scope is the - caller's ?scope (recursive) or the sitewide default. :) - let $hits := collection($scope)/*[ft:query(., $query-string, $options)] - (: Facet counts (computed while the Lucene context is intact). :) + map:merge( + for $d in $facet-dims + let $vals := distinct-values(for $p in $facet-pairs where $p?d eq $d return $p?v) + return map { $d: $vals } + ) + let $base-options := + map { + "default-operator": "and", + "filter-rewrite": "yes", + (: load the producer's display fields so ft:field can return them :) + "fields": ("site-title", "site-url") + } + (: Base result set (no facet filter) — drives the stable facet counts. + Match at document-root level (collection(…)/*): a single-step axis + preserves ft:score for field queries, unlike //*. Scope is the caller's + ?scope (recursive) or the sitewide default. :) + let $base-hits := collection($scope)/*[ft:query(., $query-string, $base-options)] let $facets := map { - "site-app": search:facet-counts($hits, "site-app"), - "site-section": search:facet-counts($hits, "site-section") + "site-app": search:facet-counts($base-hits, "site-app"), + "site-section": search:facet-counts($base-hits, "site-section") } + (: Post-filter: when a facet is selected, narrow the hits via Lucene + drill-down; otherwise the hits are the base set. :) + let $hits := + if (map:size($facet-filter) gt 0) + then collection($scope)/*[ft:query(., $query-string, map:put($base-options, "facets", $facet-filter))] + else $base-hits (: Rank by score, dedup per document (highest-scoring hit wins). :) let $ranked := for $hit in $hits diff --git a/src/test/cypress/e2e/search-facet.cy.js b/src/test/cypress/e2e/search-facet.cy.js new file mode 100644 index 0000000..f9712d8 --- /dev/null +++ b/src/test/cypress/e2e/search-facet.cy.js @@ -0,0 +1,89 @@ +const auth = { username: 'admin', password: '' }; + +// Facet drill-down on /api/search (existdb-openapi#55 / oxygen (c)): +// &facet=: (repeatable; same dim -> OR, different -> AND) +// ES post_filter semantics — selecting a value narrows the returned hits but the +// facet bucket COUNTS stay stable (computed on the base query). Uses standard +// Lucene facet drill-down (ft:query facets option), so it runs on any eXist. +// Self-contained: seeds a fixture with several site-app values. + +const APP = 'cypress-facet'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; + +const SETUP = ` +let $xconf := + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "a.xml", tuning guide content), + xmldb:store("${SCOPE}", "b.xml", more tuning content), + xmldb:store("${SCOPE}", "c.xml", tuning blog content), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/rec) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — facet drill-down (#55 / oxygen c)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('unfiltered search reports facet buckets with counts', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + expect(r.body.total).to.eq(3); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); + + it('facet=site-app:docs narrows the hits', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs`, auth }).then(r => { + expect(r.body.total).to.eq(2); + }); + }); + + it('post_filter: bucket counts stay stable when a facet is selected', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs`, auth }).then(r => { + // hits narrowed to 2, but the bucket counts still reflect the base query + expect(r.body.total).to.eq(2); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); + + it('multiple values for one dimension combine with OR', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&facet=site-app:docs&facet=site-app:blog`, auth }).then(r => { + expect(r.body.total).to.eq(3); + }); + }); + + it('the app shortcut is equivalent to facet=site-app:…', () => { + cy.request({ url: `/api/search?q=tuning&scope=${enc(SCOPE)}&app=blog`, auth }).then(r => { + expect(r.body.total).to.eq(1); + expect(r.body.facets['site-app']).to.deep.include({ docs: 2, blog: 1 }); + }); + }); +}); From 9e1b3bf5d4419f155dc9c0b5f8dc2bad2ba55963 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 11 Jun 2026 15:58:47 -0400 Subject: [PATCH 08/11] feat(search): vector-similarity branch on /api/search (#62 / oxygen d) Discovery-driven kNN: GET /api/search?vector=&similar=&k=. The client sends only the vector field + query text; the server resolves the field's embedding model from its ft:fields record (eXist-db/exist#6459), embeds the text with that model, and runs ft:query-field-vector over the scoped collection. Hits return in the existing ES-shaped envelope plus field/model/max-score. Notes: - ft:query-field-vector is context-scoped, so it is called as collection($scope)/ft:query-field-vector(...); its k is a candidate-pool hint, not a hard limit, so k is enforced via ft:score ordering + subsequence. - static vector:/ft: calls (this targets the vector-capable integration build, not a stock eXist). Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/api.json | 28 ++++++- modules/search.xqm | 88 +++++++++++++++++++++- src/test/cypress/e2e/search-vector.cy.js | 93 ++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 src/test/cypress/e2e/search-vector.cy.js diff --git a/modules/api.json b/modules/api.json index 14bbbef..e4fac9c 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3517,11 +3517,10 @@ { "name": "q", "in": "query", - "required": true, "schema": { "type": "string" }, - "description": "Search query string" + "description": "Search query string. Required for the keyword/field search; omit it when using the vector-similarity search (the `vector` + `similar` parameters)." }, { "name": "app", @@ -3582,6 +3581,31 @@ "items": { "type": "string" } }, "description": "Facet drill-down filter, ':' (e.g. site-app:docs). Repeatable: the same dimension repeated combines with OR, different dimensions with AND. ES post_filter semantics — selecting a value narrows the returned results but the facet bucket counts stay stable (they reflect the base query). The app/section parameters are shortcuts for facet=site-app:… / facet=site-section:…" + }, + { + "name": "vector", + "in": "query", + "schema": { + "type": "string" + }, + "description": "Switch to vector-similarity (kNN) search over this named vector field (a `field` value from GET /api/search/fields whose `kind` is `vector`). Requires `similar`; `q` is ignored. The field's embedding model is resolved server-side from its ft:fields record, so the client sends only the field and the text. Subject to the same field-level security as `field`." + }, + { + "name": "similar", + "in": "query", + "schema": { + "type": "string" + }, + "description": "The query text for vector search: it is embedded with the vector field's configured model and matched by similarity. Used only with `vector`." + }, + { + "name": "k", + "in": "query", + "schema": { + "type": "integer", + "default": 10 + }, + "description": "Number of nearest-neighbor results to return for vector search (default 10). Used only with `vector`." } ], "responses": { diff --git a/modules/search.xqm b/modules/search.xqm index 77e2593..1d2e6f6 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -27,6 +27,9 @@ import module namespace roaster="http://e-editiones.org/roaster"; import module namespace kwic="http://exist-db.org/xquery/kwic"; declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization"; +(: Vector module (the embedding/kNN extension). Static dependency: this build + : targets the vector-capable integration instance, not a stock eXist. :) +declare namespace vector="http://exist-db.org/xquery/vector"; declare option output:method "json"; declare option output:media-type "application/json"; @@ -97,6 +100,82 @@ declare %private function search:facet-counts($hits as node()*, $dimension as xs try { ft:facets($hits, $dimension, ()) } catch * { map {} } }; +(:~ + : Vector-similarity branch of /api/search. + : GET /api/search?vector=&similar=&k=[&scope=] + : + : Discovery-driven: the field's embedding model is read from its ft:fields record + : (the `model` property, present on text-embedding vector fields, per + : eXist-db/exist#6459), so the client sends only {field, text}. The text is + : embedded with that model and run as a kNN over the field; hits come back in the + : same ES-shaped envelope as the keyword search, plus `field`/`model`/`max-score`. + : + : Notes: + : - ft:query-field-vector is context-scoped (it resolves against the documents in + : the focus), so it is called as collection($scope)/ft:query-field-vector(...). + : - the engine's k is a candidate-pool hint, not a hard limit, so k is enforced + : here via ft:score ordering + subsequence (same as keyword pagination). + :) +declare %private function search:vector-query( + $field as xs:string, $similar as xs:string?, $scope as xs:string+, + $k as xs:integer, $groups as xs:string*, $is-dba as xs:boolean +) { + if (empty($similar) or $similar = "") + then roaster:response(400, "application/json", + map { "error": "Missing required parameter for vector search: similar" }) + (: Field-level security: the same policy /api/search/fields applies. :) + else if (not(fpol:visible($field, $groups, $is-dba))) + then roaster:response(403, "application/json", + map { "error": "Field not available: " || $field }) + else + (: Resolve the field's embedding model from its ft:fields record. Bind $r + explicitly (avoid ?key in a predicate — eXist mis-handles the cardinality + for >1 item, XPTY0004). :) + let $vrec := (for $r in ft:fields($scope) where $r?kind = "vector" and $r?field = $field return $r)[1] + let $model := $vrec?model + return + if (empty($vrec)) + then roaster:response(404, "application/json", + map { "error": "Vector field not found in scope: " || $field }) + else if (empty($model) or $model = "") + then roaster:response(400, "application/json", + map { "error": "Field '" || $field || "' has no embedding model; it cannot embed query text (index it with a model, or query with a precomputed vector)" }) + else + let $vec := vector:embed($similar, $model) + let $hits := collection($scope)/ft:query-field-vector($field, $vec, $k) + let $ranked := + for $h in $hits + let $score := ft:score($h) + order by $score descending + return map { "hit": $h, "score": $score, "uri": document-uri(root($h)) } + let $top := subsequence($ranked, 1, $k) + return map { + "query": $similar, + "field": $field, + "model": $model, + "total": count($ranked), + "k": $k, + "max-score": ($top[1]?score, 0)[1], + "results": array { + for $m in $top + let $hit := $m?hit + let $doc-uri := $m?uri + let $app := replace($doc-uri, "^/db/apps/([^/]+)/.*$", "$1") + return map { + "uri": $doc-uri, + "path": $doc-uri, + "title": (string($hit/ancestor-or-self::*[title][1]/title)[. ne ""], "(untitled)")[1], + "app": $app, + "url": site:resolve-link($app, replace($doc-uri, "^/db/apps/[^/]+", "")), + "score": $m?score, + "snippet": serialize( + { substring(string-join($hit//text(), " "), 1, 200) }, + map { "method": "xml" }) + } + } + } +}; + (:~ : Sitewide search over the shared `site-content` field. : GET /api/search?q=array:count&app=docs§ion=functions&limit=20&offset=0 @@ -132,8 +211,15 @@ declare function search:query($request as map(*)) { let $is-dba := ($user?dba, false())[1] let $limit := ($request?parameters?limit, 20)[1] cast as xs:integer let $offset := ($request?parameters?offset, 0)[1] cast as xs:integer + (: vector: switch to similarity search over a named vector field. similar: the + query text to embed (server resolves the field's model). Mutually exclusive + with the keyword path — when present, q is not required. :) + let $vector-field := $request?parameters?vector[. ne ""] + let $k := ($request?parameters?k, 10)[1] cast as xs:integer return - if (empty($q) or $q = "") + if (exists($vector-field)) + then search:vector-query($vector-field, $request?parameters?similar, $scope, $k, $groups, $is-dba) + else if (empty($q) or $q = "") then map { "error": "Missing required parameter: q" } else if (exists($field) and not(fpol:visible($field, $groups, $is-dba))) (: Field-level security: the same policy /api/search/fields applies — a diff --git a/src/test/cypress/e2e/search-vector.cy.js b/src/test/cypress/e2e/search-vector.cy.js new file mode 100644 index 0000000..827ce03 --- /dev/null +++ b/src/test/cypress/e2e/search-vector.cy.js @@ -0,0 +1,93 @@ +const auth = { username: 'admin', password: '' }; + +// Vector-similarity search on /api/search (existdb-openapi#62 / oxygen (d)): +// &vector=&similar=&k= +// Discovery-driven: the client sends only the vector field + query text; the +// server resolves the field's embedding model from its ft:fields record +// (eXist-db/exist#6459) and embeds the text before the kNN. Requires a +// vector-capable eXist (the vector extension module + a local embedding model), +// so it runs on the trio/integration instance, not a stock eXist. +// Self-contained: seeds a small corpus with a text-embedding vector field. + +const APP = 'cypress-vector'; +const SCOPE = `/db/apps/${APP}`; +const CONF = `/db/system/config/db/apps/${APP}`; +const MODEL = 'all-MiniLM-L6-v2'; + +const SETUP = ` +let $xconf := + + + + + + + + + +return ( + xmldb:create-collection("/db/system/config/db/apps", "${APP}"), + xmldb:store("${CONF}", "collection.xconf", $xconf), + xmldb:create-collection("/db/apps", "${APP}"), + xmldb:store("${SCOPE}", "speed.xml", Query performance tuningMake your database queries run much faster by optimizing indexes and caching.), + xmldb:store("${SCOPE}", "cooking.xml", Pasta recipesHow to cook delicious Italian pasta with a fresh tomato and basil sauce.), + xmldb:store("${SCOPE}", "weather.xml", Weather patternsUnderstanding seasonal climate changes and rainfall in the tropics.), + xmldb:reindex("${SCOPE}"), + "indexed=" || count(collection("${SCOPE}")/doc) +)[last()] +`; + +const TEARDOWN = ` +(if (xmldb:collection-available("${SCOPE}")) then xmldb:remove("${SCOPE}") else (), + if (xmldb:collection-available("${CONF}")) then xmldb:remove("${CONF}") else (), + "cleaned")[last()] +`; + +function runAdmin(query) { + return cy.request({ url: '/api/query', method: 'POST', auth, body: { query } }).then(r => { + if (r.body && r.body.cursor) cy.request({ url: `/api/query/${r.body.cursor}`, method: 'DELETE', auth, failOnStatusCode: false }); + }); +} + +const enc = encodeURIComponent; + +describe('GET /api/search — vector similarity (#62 / oxygen d)', () => { + before(() => runAdmin(SETUP)); + after(() => runAdmin(TEARDOWN)); + + it('ranks by semantic similarity (client sends only field + text)', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('how do I speed up my database queries')}&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.status).to.eq(200); + // the performance doc is most similar to the query, despite no shared keywords with "speed up" + expect(r.body.results[0].uri).to.eq(`${SCOPE}/speed.xml`); + expect(r.body.results[0].score).to.be.greaterThan(0); + }); + }); + + it('resolves and echoes the field model server-side (no model sent by client)', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('fast queries')}&scope=${enc(SCOPE)}`, auth }).then(r => { + expect(r.body.field).to.eq('site-embedding'); + expect(r.body.model).to.eq(MODEL); + expect(r.body['max-score']).to.eq(r.body.results[0].score); + }); + }); + + it('k limits the number of results', () => { + cy.request({ url: `/api/search?vector=site-embedding&similar=${enc('fast queries')}&scope=${enc(SCOPE)}&k=2`, auth }).then(r => { + expect(r.body.k).to.eq(2); + expect(r.body.results).to.have.length(2); + }); + }); + + it('missing similar -> 400', () => { + cy.request({ url: `/api/search?vector=site-embedding&scope=${enc(SCOPE)}`, auth, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(400); + }); + }); + + it('unknown vector field -> 404', () => { + cy.request({ url: `/api/search?vector=no-such-field&similar=${enc('anything')}&scope=${enc(SCOPE)}`, auth, failOnStatusCode: false }).then(r => { + expect(r.status).to.eq(404); + }); + }); +}); From d98ef50295d34a040049519a3fa6187f95f63dee Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 11 Jun 2026 23:05:43 -0400 Subject: [PATCH 09/11] feat(search): vector total=count(results) + clamp k to 1-100 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - total now reflects the number of results returned (= k, or fewer if the corpus is smaller), instead of the kNN's over-returned candidate pool (the pool is the symptom of the eXist-core ft:query-field-vector k bug; the server-side re-rank + subsequence already returns exactly k). - k is clamped to [1, 100] (default 10) — a kNN result count, not paging. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/api.json | 6 ++++-- modules/search.xqm | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/api.json b/modules/api.json index e4fac9c..2fbef2c 100644 --- a/modules/api.json +++ b/modules/api.json @@ -3603,9 +3603,11 @@ "in": "query", "schema": { "type": "integer", - "default": 10 + "default": 10, + "minimum": 1, + "maximum": 100 }, - "description": "Number of nearest-neighbor results to return for vector search (default 10). Used only with `vector`." + "description": "Number of nearest-neighbor results to return for vector search (default 10, clamped to 1–100). The response's `total` equals the number of results returned. Used only with `vector`." } ], "responses": { diff --git a/modules/search.xqm b/modules/search.xqm index 1d2e6f6..0ad9968 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -153,7 +153,12 @@ declare %private function search:vector-query( "query": $similar, "field": $field, "model": $model, - "total": count($ranked), + (: total = results returned (= k, or fewer if the corpus is + smaller). NB: the kNN's own k currently over-returns a + candidate pool — tracked by the eXist-core fix to + ft:query-field-vector — so count the post-rank/subsequence + slice, not the raw pool. :) + "total": count($top), "k": $k, "max-score": ($top[1]?score, 0)[1], "results": array { @@ -215,7 +220,8 @@ declare function search:query($request as map(*)) { query text to embed (server resolves the field's model). Mutually exclusive with the keyword path — when present, q is not required. :) let $vector-field := $request?parameters?vector[. ne ""] - let $k := ($request?parameters?k, 10)[1] cast as xs:integer + (: clamp k to [1, 100]: default 10, hard cap 100 (a kNN result count, not paging) :) + let $k := max((1, min((($request?parameters?k, 10)[1] cast as xs:integer, 100)))) return if (exists($vector-field)) then search:vector-query($vector-field, $request?parameters?similar, $scope, $k, $groups, $is-dba) From 5fb283d08d1976d510237630772fa785a7f6d4f6 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 11 Jun 2026 23:32:18 -0400 Subject: [PATCH 10/11] feat(search): vector kNN via ft:query-vector node form (k authoritative) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the vector branch from collection($scope)/ft:query-field-vector($field, …) to ft:query-vector(collection($scope)//, …, k), targeting the vector field's indexed element (read from ft:fields discovery alongside the model). The field form is evaluated per-document in a path step (a 1-doc kNN per node, unioned) — so it ignores k AND scales poorly; the node form does one true cross-document top-k with k authoritative server-side (root-cause + fix handed off by exist-strategy; the field-targeted core fix is tracked separately with Duncan). Client contract unchanged (still ?vector=, discovery-driven). The order-by-ft:score + subsequence is kept (explicit ranking, robust). Caveat noted in-code: an element with >1 vector field targets the first; revisit when the core ft:query-field-vector fix lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/search.xqm | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/modules/search.xqm b/modules/search.xqm index 0ad9968..6fb157a 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -104,17 +104,19 @@ declare %private function search:facet-counts($hits as node()*, $dimension as xs : Vector-similarity branch of /api/search. : GET /api/search?vector=&similar=&k=[&scope=] : - : Discovery-driven: the field's embedding model is read from its ft:fields record - : (the `model` property, present on text-embedding vector fields, per + : Discovery-driven: the field's embedding model AND indexed element are read from + : its ft:fields record (the `model` and `element` properties, per : eXist-db/exist#6459), so the client sends only {field, text}. The text is - : embedded with that model and run as a kNN over the field; hits come back in the - : same ES-shaped envelope as the keyword search, plus `field`/`model`/`max-score`. + : embedded with that model and run as a kNN over the field's element; hits come + : back in the same ES-shaped envelope as keyword search, plus `field`/`model`/ + : `max-score`. : : Notes: - : - ft:query-field-vector is context-scoped (it resolves against the documents in - : the focus), so it is called as collection($scope)/ft:query-field-vector(...). - : - the engine's k is a candidate-pool hint, not a hard limit, so k is enforced - : here via ft:score ordering + subsequence (same as keyword pagination). + : - Uses the node-arg form ft:query-vector(collection($scope)//, vec, k), + : which applies a true cross-document top-k with k authoritative server-side; the + : field form ft:query-field-vector is evaluated per-document in a path step + : (ignores k, scales poorly — eXist-core bug, fix tracked separately). + : - ft:score ordering + subsequence is kept for explicit, robust ranking. :) declare %private function search:vector-query( $field as xs:string, $similar as xs:string?, $scope as xs:string+, @@ -133,6 +135,7 @@ declare %private function search:vector-query( for >1 item, XPTY0004). :) let $vrec := (for $r in ft:fields($scope) where $r?kind = "vector" and $r?field = $field return $r)[1] let $model := $vrec?model + let $element := $vrec?element return if (empty($vrec)) then roaster:response(404, "application/json", @@ -142,7 +145,18 @@ declare %private function search:vector-query( map { "error": "Field '" || $field || "' has no embedding model; it cannot embed query text (index it with a model, or query with a precomputed vector)" }) else let $vec := vector:embed($similar, $model) - let $hits := collection($scope)/ft:query-field-vector($field, $vec, $k) + (: Use the node-arg form ft:query-vector(nodes, vec, k), which applies a + true cross-document top-k with k authoritative server-side. The field + form collection($scope)/ft:query-field-vector($field, ...) is evaluated + per-document (a 1-doc kNN per node, unioned), so it both ignores k and + scales poorly — a known eXist-core bug, fix tracked separately. We + target the vector field's indexed element (from ft:fields discovery); + ft:query-vector resolves the field from that element's index config. + Caveat: if an element carries >1 vector field this targets the first, + not necessarily $field — fine for one-field-per-element corpora; when + the core ft:query-field-vector fix lands, switch back to the + field-targeted form for precise multi-field selection. :) + let $hits := ft:query-vector(collection($scope)//*[local-name() = $element], $vec, $k) let $ranked := for $h in $hits let $score := ft:score($h) From 042763280bcf257b1f55f440c4d2ad59e75b8396 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 12 Jun 2026 08:02:06 -0400 Subject: [PATCH 11/11] fix(search): keep KWIC highlighting under facet drill-down (#55 / oxygen c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The facets-option ft:query (Lucene drill-down) doesn't collect per-match offsets, so its result nodes can't drive ft:highlight-field-matches/KWIC — the faceted path returned a full-body snippet with no and empty highlights. Intersect the drill set with $base-hits (which carry the match data) by node identity, so the returned hits are the match-bearing nodes narrowed to the facet selection. Preserves post_filter narrowing + the base-query facet counts. Reproduced + fix verified on :19110: faceted hit went from 0 to 2 highlight matches; counts unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/search.xqm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/search.xqm b/modules/search.xqm index 77e2593..bfb2ace 100644 --- a/modules/search.xqm +++ b/modules/search.xqm @@ -191,10 +191,16 @@ declare function search:query($request as map(*)) { "site-section": search:facet-counts($base-hits, "site-section") } (: Post-filter: when a facet is selected, narrow the hits via Lucene - drill-down; otherwise the hits are the base set. :) + drill-down; otherwise the hits are the base set. The facets-option + query does NOT collect per-match offsets, so its nodes can't drive + ft:highlight-field-matches/KWIC — intersect the drill set with + $base-hits (which carry the match data) so the returned nodes are the + match-bearing ones, narrowed to the facet selection. (Identity + intersection; preserves the post_filter narrowing and the base-query + facet counts.) :) let $hits := if (map:size($facet-filter) gt 0) - then collection($scope)/*[ft:query(., $query-string, map:put($base-options, "facets", $facet-filter))] + then $base-hits intersect collection($scope)/*[ft:query(., $query-string, map:put($base-options, "facets", $facet-filter))] else $base-hits (: Rank by score, dedup per document (highest-scoring hit wins). :) let $ranked :=