Skip to content

Commit 84e129e

Browse files
authored
Merge pull request #479 from esokullu/main
Fix and verify find_text selection and pending upload evidence
2 parents 2260130 + 6fefd56 commit 84e129e

9 files changed

Lines changed: 472 additions & 41 deletions

File tree

src/chrome/src/agent/agent.js

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,7 +1352,7 @@ export class Agent {
13521352
}
13531353

13541354
_findTextMatchLoopIdentity(result) {
1355-
if (result?.success !== true || !result?.rect || typeof result.rect !== 'object') return '';
1355+
if (result?.success !== true || result?.verified === false || !result?.rect || typeof result.rect !== 'object') return '';
13561356
const rect = result.rect;
13571357
const pageX = typeof rect.pageX === 'number' ? rect.pageX : NaN;
13581358
const pageY = typeof rect.pageY === 'number' ? rect.pageY : NaN;
@@ -1362,10 +1362,23 @@ export class Agent {
13621362
const height = typeof rect.height === 'number' ? rect.height : NaN;
13631363
const x = Number.isFinite(pageX) ? pageX : viewportX;
13641364
const y = Number.isFinite(pageY) ? pageY : viewportY;
1365-
if (![x, y, width, height].every(Number.isFinite)) return '';
1366-
return [x, y, width, height]
1365+
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return '';
1366+
let selectionIdentity = 'document';
1367+
if (result.selectionSource === 'text_control') {
1368+
const selectionStart = result.selectionStart;
1369+
const selectionEnd = result.selectionEnd;
1370+
if (
1371+
!Number.isInteger(selectionStart)
1372+
|| !Number.isInteger(selectionEnd)
1373+
|| selectionStart < 0
1374+
|| selectionEnd <= selectionStart
1375+
) return '';
1376+
selectionIdentity = `text_control:${selectionStart}:${selectionEnd}`;
1377+
}
1378+
const rectIdentity = [x, y, width, height]
13671379
.map(value => Math.round(value * 2) / 2)
13681380
.join(',');
1381+
return `${selectionIdentity}|${rectIdentity}`;
13691382
}
13701383

13711384
_noteHealthyLoopCall(tabId) {
@@ -10924,9 +10937,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
1092410937
&& this.constructor.EXECUTION_APP_STATE_TOOLS.has(name);
1092510938
const requiredScheduleSucceeded = state?.requiredSchedulingTool === name
1092610939
&& this._isSuccessfulSchedulingEvidence(result);
10940+
// find_text is observational: verified:false means it did not prove a
10941+
// visible selection. Keep this tool-specific because mutations such as
10942+
// upload_file may dispatch successfully, return verified:false, and rely
10943+
// on the completion invariant's required follow-up page observation.
10944+
const unverifiedFindText = name === 'find_text'
10945+
&& (result?.found !== true || result?.verified !== true || result?.inconclusive === true);
1092710946
if (!state?.enabled
1092810947
|| name === 'done'
1092910948
|| (this.constructor.EXECUTION_META_TOOLS.has(name) && !requestedAppStateTool)
10949+
|| unverifiedFindText
1093010950
|| (!this._isSuccessfulExecutionEvidence(result) && !requiredScheduleSucceeded)) return;
1093110951
state.successfulTaskToolCalls += 1;
1093210952
if (requiredScheduleSucceeded) state.successfulRequiredSchedulingToolCalls += 1;

src/chrome/src/agent/planner.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Rules:
6767
memory: scratchpad_write, progress_update, progress_read
6868
schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event)
6969
finish: done
70-
- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To locate and highlight literal page text, plan find_text instead of Ctrl/Cmd+F.
70+
- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To select one literal page-text match, plan find_text instead of Ctrl/Cmd+F. Each find_text call replaces the previous selection and does not open browser Find UI; never plan sequential calls as simultaneous highlights.
7171
- For repeated same-kind UI mutations (for example following many users), plan visible UI first with bounded batches, verification, progress_update, and wait_for_stable pacing; do not plan one huge same-shape click/tool batch.
7272
- Do not invent a prerequisite to discover a raw identifier (email address, account ID, username, or similar) when the target UI provides a name-based contact/entity picker and the user already supplied a human-readable name. Plan to use the picker first. Inspect surrounding pages or messages for the raw identifier only if the picker fails, returns multiple ambiguous matches, or the user explicitly asked for the identifier itself.
7373
- Set confidence from 0.0 to 1.0 for how clear and safe this plan is. Use 0.90+ only when the task, page state, and next steps are straightforward; use lower scores for ambiguity, destructive changes, payments, credentials, bulk mutations, or uncertain page state.
@@ -124,7 +124,7 @@ Rules:
124124
- schedule_task supports one-shot times and fixed-minute intervals only. Calendar/cron recurrence such as monthly is unsupported: classify it as clarify, explain the limitation in localized.summary, and ask for a one-shot time or fixed interval. Never convert calendar recurrence into an approximate interval.
125125
- Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale.
126126
- For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty.
127-
- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan modifier combinations or browser UI shortcuts; use find_text to locate and highlight page text instead of Ctrl/Cmd+F.
127+
- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI.
128128
- Do not invent URLs, credentials, tool names, or facts.`;
129129

130130
export function normalizePlannerLocale(value) {

src/chrome/src/agent/tools.js

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ export const AGENT_TOOLS = [
298298
type: 'function',
299299
function: {
300300
name: 'press_keys',
301-
description: 'Press one unmodified keyboard key. Supports only Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, and ArrowRight. Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to locate and highlight page text.',
301+
description: 'Press one unmodified keyboard key. Supports only Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, and ArrowRight. Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to select one page-text match.',
302302
parameters: {
303303
type: 'object',
304304
properties: {
@@ -512,7 +512,7 @@ export const AGENT_TOOLS = [
512512
type: 'function',
513513
function: {
514514
name: 'find_text',
515-
description: 'Find and select/highlight the next occurrence of literal text in the current page. Use this instead of Ctrl+F or Cmd+F; press_keys cannot send modifier combinations or open browser UI. Repeating the same search advances to the next match.',
515+
description: 'Find and select the next occurrence of literal text in the current page. Use this instead of Ctrl+F or Cmd+F; press_keys cannot send modifier combinations. Repeating the same search advances to the next match. Each call replaces the previous page selection, so only the current match remains selected. This tool does not open the browser Find UI. Never claim that sequential find_text calls leave multiple terms highlighted.',
516516
parameters: {
517517
type: 'object',
518518
properties: {
@@ -1432,7 +1432,7 @@ Available tools:
14321432
- schedule_resume: Durably pause this current task and resume it later in the same tab/conversation. Terminal tool; use only for external waits.
14331433
- schedule_task: Create a one-shot or fixed-minute-interval task only when the user explicitly asks for future scheduled work. It does not support calendar/cron recurrence; never approximate monthly recurrence. Prefer URL targets for repeatable automations; current_tab is strict and fails if the tab changes.
14341434
- get_selection: Get highlighted text
1435-
- find_text: Find and select/highlight literal page text. Use this instead of Ctrl/Cmd+F.
1435+
- find_text: Select one literal page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection; it does not open browser Find UI or keep multiple terms highlighted.
14361436
- press_keys: Press only unmodified Escape/Tab/Enter/arrows. Modifier combinations and browser shortcuts are unsupported.
14371437
- new_tab: Open a background reference tab; the current run stays on its original tab
14381438
- clarify: Pause and ask the user a question. Use ONLY for material ambiguity that you cannot resolve by reading the page (e.g. "my API key" on a site with multiple plugins that each have one). Unanswered clarifies auto-select options[0] after the timeout (default 60s) with source=timeout (not high-risk approval); Settings Instant yields source=auto (intentional auto-approve — continue). Put the safe/default first. Do NOT use to confirm correct actions; do NOT call before every step. Budget 1-2 per run, max.
@@ -1704,7 +1704,7 @@ TOOLS — use ONLY these:
17041704
- click({text}): Click by visible text. Fallback when no ref_id.
17051705
- type_text({text}): Type into the focused element. Click the field first.
17061706
- get_selection: Read highlighted text.
1707-
- find_text({text}): Locate and highlight literal page text instead of using Ctrl/Cmd+F.
1707+
- find_text({text}): Select one literal page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection; no browser Find UI or simultaneous highlights.
17081708
- press_keys({key}): Press one supported unmodified key. Ctrl/Cmd/Alt/Shift combinations and browser shortcuts are unavailable.
17091709
- navigate({url}): Go to a URL.
17101710
- new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround.
@@ -1774,7 +1774,7 @@ TOOLS — use only these:
17741774
- click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes.
17751775
- read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run.
17761776
- get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows, never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts.
1777-
- extract_data: tables/headings/images/links. get_selection: read highlighted text. find_text({text}): locate and highlight literal page text instead of Ctrl/Cmd+F. read_pdf: read a PDF.
1777+
- extract_data: tables/headings/images/links. get_selection: read highlighted text. find_text({text}): select one literal page-text match; each call replaces the previous selection and never creates simultaneous highlights or browser Find UI. read_pdf: read a PDF.
17781778
- wait_for_element({selector}) / wait_for_stable({quietMs}): wait for an element / for the page to go quiet after an action.
17791779
- schedule_resume({after_seconds|run_at, reason, resume_instruction}): terminal durable pause for this current task.
17801780
- schedule_task({title, prompt, schedule, target, mode}): create one-shot or fixed-minute-interval future work only when explicitly requested by the user. Calendar/cron recurrence is unsupported and must not be approximated. Prefer target.type:"url" for monitors/repeatable automations; use current_tab only for exact current-tab state.

src/chrome/src/content/content.js

Lines changed: 79 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1849,9 +1849,10 @@
18491849
const matchCase = params?.matchCase === true;
18501850
const backwards = params?.backwards === true;
18511851
const wrap = params?.wrap !== false;
1852-
// Match browser Find semantics across the whole document, including
1853-
// embedded frames. Without searchInFrames, find_text falsely reports a
1854-
// miss for text that Ctrl/Cmd+F would find inside an iframe.
1852+
// Match browser Find semantics across the whole page, including frames.
1853+
// When the active match moves into a frame, the top document can retain
1854+
// an older selection; frame focus keeps that stale range from verifying
1855+
// the current hit.
18551856
const found = window.find(text, matchCase, backwards, wrap, false, true, false);
18561857
if (!found) {
18571858
return {
@@ -1863,11 +1864,56 @@
18631864
error: `find_text: "${text}" was not found on the current page. Re-read the page or try a shorter literal phrase.`,
18641865
};
18651866
}
1867+
const activeElement = window.document?.activeElement;
1868+
const activeElementTag = String(activeElement?.tagName || '').toUpperCase();
1869+
const inputType = String(activeElement?.type || 'text').toLowerCase();
1870+
const isTextControl = activeElementTag === 'TEXTAREA'
1871+
|| (activeElementTag === 'INPUT' && ['text', 'search', 'url', 'tel', 'email'].includes(inputType));
1872+
const normalizedQuery = text.normalize('NFC');
1873+
const matchesQuery = (value) => {
1874+
const normalizedValue = String(value || '').normalize('NFC');
1875+
return matchCase
1876+
? normalizedValue === normalizedQuery
1877+
: normalizedValue.toLowerCase() === normalizedQuery.toLowerCase();
1878+
};
1879+
const hasVisibleBounds = (candidate) => !!candidate
1880+
&& [candidate.x, candidate.y, candidate.width, candidate.height].every(Number.isFinite)
1881+
&& candidate.width > 0
1882+
&& candidate.height > 0;
18661883
const selection = window.getSelection?.();
1867-
const selectedText = String(selection?.toString?.() || '');
1884+
const documentSelectedText = String(selection?.toString?.() || '');
1885+
const documentBounds = selection?.rangeCount
1886+
? selection.getRangeAt(0).getBoundingClientRect()
1887+
: undefined;
1888+
let controlSelectedText = '';
1889+
let controlBounds;
1890+
const controlSelectionStart = activeElement?.selectionStart;
1891+
const controlSelectionEnd = activeElement?.selectionEnd;
1892+
if (
1893+
isTextControl
1894+
&& Number.isInteger(controlSelectionStart)
1895+
&& Number.isInteger(controlSelectionEnd)
1896+
&& controlSelectionStart >= 0
1897+
&& controlSelectionEnd > controlSelectionStart
1898+
) {
1899+
controlSelectedText = String(activeElement.value || '').slice(controlSelectionStart, controlSelectionEnd);
1900+
controlBounds = activeElement.getBoundingClientRect?.();
1901+
}
1902+
const documentMatchesQuery = matchesQuery(documentSelectedText);
1903+
const controlMatchesQuery = matchesQuery(controlSelectedText);
1904+
let selectedText = documentSelectedText;
1905+
let selectionSource = 'document';
1906+
let bounds = documentBounds;
1907+
if (documentMatchesQuery && hasVisibleBounds(documentBounds)) {
1908+
// Keep the page selection. A focused field can retain an unrelated
1909+
// selection while window.find advances to a normal document match.
1910+
} else if (controlMatchesQuery && hasVisibleBounds(controlBounds)) {
1911+
selectedText = controlSelectedText;
1912+
bounds = controlBounds;
1913+
selectionSource = 'text_control';
1914+
}
18681915
let rect;
1869-
if (selection?.rangeCount) {
1870-
const bounds = selection.getRangeAt(0).getBoundingClientRect();
1916+
if (bounds) {
18711917
const scrollX = Number.isFinite(Number(window.scrollX)) ? Number(window.scrollX) : 0;
18721918
const scrollY = Number.isFinite(Number(window.scrollY)) ? Number(window.scrollY) : 0;
18731919
rect = {
@@ -1879,12 +1925,39 @@
18791925
height: bounds.height,
18801926
};
18811927
}
1928+
const hasVisibleRect = !!rect
1929+
&& [rect.x, rect.y, rect.pageX, rect.pageY, rect.width, rect.height].every(Number.isFinite)
1930+
&& rect.width > 0
1931+
&& rect.height > 0;
1932+
const selectionMatchesQuery = matchesQuery(selectedText);
1933+
const selectionInFrame = activeElementTag === 'IFRAME' || activeElementTag === 'FRAME';
1934+
const hasSelectionIdentity = selectionSource !== 'text_control'
1935+
|| (
1936+
Number.isInteger(controlSelectionStart)
1937+
&& Number.isInteger(controlSelectionEnd)
1938+
&& controlSelectionStart >= 0
1939+
&& controlSelectionEnd > controlSelectionStart
1940+
);
1941+
const verified = selectionMatchesQuery && hasVisibleRect && hasSelectionIdentity && !selectionInFrame;
18821942
return {
18831943
success: true,
18841944
found: true,
1945+
verified,
18851946
query: text,
18861947
selectedText,
1948+
selectionMatchesQuery,
1949+
selectionSource,
1950+
selectionInFrame,
1951+
selectionScope: selectionInFrame ? 'frame_match_unverified' : 'current_match_only',
1952+
...(selectionSource === 'text_control'
1953+
? { selectionStart: controlSelectionStart, selectionEnd: controlSelectionEnd }
1954+
: {}),
1955+
replacesPreviousSelection: !selectionInFrame,
1956+
browserFindUiOpened: false,
18871957
...(rect ? { rect } : {}),
1958+
warning: verified
1959+
? 'Only this match is selected. This call replaced any previous page selection, and it did not open the browser Find UI. Do not claim earlier find_text matches remain highlighted.'
1960+
: 'window.find reported a match, but WebBrain could not verify a visible current selection in the top document (for example, the active match may be inside a frame while an older top-document selection remains). Do not claim it is visibly highlighted. The browser Find UI was not opened.',
18881961
};
18891962
} catch (error) {
18901963
return { success: false, found: false, dispatched: false, noDispatch: true, error: `find_text failed: ${error.message || error}` };

src/firefox/src/agent/agent.js

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,7 +1380,7 @@ export class Agent {
13801380
}
13811381

13821382
_findTextMatchLoopIdentity(result) {
1383-
if (result?.success !== true || !result?.rect || typeof result.rect !== 'object') return '';
1383+
if (result?.success !== true || result?.verified === false || !result?.rect || typeof result.rect !== 'object') return '';
13841384
const rect = result.rect;
13851385
const pageX = typeof rect.pageX === 'number' ? rect.pageX : NaN;
13861386
const pageY = typeof rect.pageY === 'number' ? rect.pageY : NaN;
@@ -1390,10 +1390,23 @@ export class Agent {
13901390
const height = typeof rect.height === 'number' ? rect.height : NaN;
13911391
const x = Number.isFinite(pageX) ? pageX : viewportX;
13921392
const y = Number.isFinite(pageY) ? pageY : viewportY;
1393-
if (![x, y, width, height].every(Number.isFinite)) return '';
1394-
return [x, y, width, height]
1393+
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return '';
1394+
let selectionIdentity = 'document';
1395+
if (result.selectionSource === 'text_control') {
1396+
const selectionStart = result.selectionStart;
1397+
const selectionEnd = result.selectionEnd;
1398+
if (
1399+
!Number.isInteger(selectionStart)
1400+
|| !Number.isInteger(selectionEnd)
1401+
|| selectionStart < 0
1402+
|| selectionEnd <= selectionStart
1403+
) return '';
1404+
selectionIdentity = `text_control:${selectionStart}:${selectionEnd}`;
1405+
}
1406+
const rectIdentity = [x, y, width, height]
13951407
.map(value => Math.round(value * 2) / 2)
13961408
.join(',');
1409+
return `${selectionIdentity}|${rectIdentity}`;
13971410
}
13981411

13991412
_noteHealthyLoopCall(tabId) {
@@ -9756,9 +9769,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
97569769
&& this.constructor.EXECUTION_APP_STATE_TOOLS.has(name);
97579770
const requiredScheduleSucceeded = state?.requiredSchedulingTool === name
97589771
&& this._isSuccessfulSchedulingEvidence(result);
9772+
// find_text is observational: verified:false means it did not prove a
9773+
// visible selection. Keep this tool-specific because mutations such as
9774+
// upload_file may dispatch successfully, return verified:false, and rely
9775+
// on the completion invariant's required follow-up page observation.
9776+
const unverifiedFindText = name === 'find_text'
9777+
&& (result?.found !== true || result?.verified !== true || result?.inconclusive === true);
97599778
if (!state?.enabled
97609779
|| name === 'done'
97619780
|| (this.constructor.EXECUTION_META_TOOLS.has(name) && !requestedAppStateTool)
9781+
|| unverifiedFindText
97629782
|| (!this._isSuccessfulExecutionEvidence(result) && !requiredScheduleSucceeded)) return;
97639783
state.successfulTaskToolCalls += 1;
97649784
if (requiredScheduleSucceeded) state.successfulRequiredSchedulingToolCalls += 1;

0 commit comments

Comments
 (0)