diff --git a/computer-use-linux/src/server.rs b/computer-use-linux/src/server.rs index ed3456683..5326ecdc4 100644 --- a/computer-use-linux/src/server.rs +++ b/computer-use-linux/src/server.rs @@ -15,6 +15,7 @@ use crate::screenshot::{ capture_screenshot_raw, prepare_screenshot_payload, RawScreenshotCapture, ScreenshotCapture, ScreenshotOutputFormat, ScreenshotPayloadOptions, }; +use crate::terminal::uses_terminal_paste_shortcut; use crate::windowing::registry; use crate::windows::{ focus_window_target, focused_window, list_windows, resolve_window_target, @@ -1684,7 +1685,8 @@ impl ComputerUseLinux { let received = Some(serde_json::json!(params.clone())); let mut input_guard = InputOperationGuard::new(Arc::clone(&self.input_operation_lock).lock_owned().await); - let focus = match self.focus_target_for_input(¶ms.window_target()).await { + let window_target = params.window_target(); + let focus = match self.focus_target_for_input(&window_target).await { Ok(focus) => focus, Err(message) => { return Json(ActionOutput { @@ -1699,6 +1701,24 @@ impl ComputerUseLinux { if self.should_prefer_kde_clipboard_text_backend() { match self.ensure_portal_keyboard_session().await { Ok(Some(session)) => { + let kde_focus = if window_target.has_target() { + match self.focus_target_for_input(&window_target).await { + Ok(focus) => focus, + Err(message) => { + return Json(ActionOutput { + ok: false, + implemented: true, + action: "type_text".to_string(), + message, + received, + }); + } + } + } else { + focus.clone() + }; + let use_terminal_paste = + kde_clipboard_uses_terminal_paste(&window_target, kde_focus.as_ref()).await; let clipboard_guard = Arc::clone(&self.kde_clipboard_lock).lock_owned().await; let session_for_input = session.clone(); let text = params.text.clone(); @@ -1708,6 +1728,7 @@ impl ComputerUseLinux { run_kde_clipboard_paste_text( &session_for_input, &text, + use_terminal_paste, portal_operation_guard, ) .await @@ -1720,7 +1741,7 @@ impl ComputerUseLinux { "type_text", Err(guarded_result.unwrap_err()), received, - focus, + kde_focus, )); } }; @@ -1729,13 +1750,13 @@ impl ComputerUseLinux { let result = guarded_result.expect("guarded task returned its guards"); match result { Ok(message) => { - let notes = self.input_landing_notes(focus.as_ref(), true).await; + let notes = self.input_landing_notes(kde_focus.as_ref(), true).await; return Json(with_notes( successful_action_with_focus( "type_text", &message, received, - focus, + kde_focus, ), notes, )); @@ -1749,7 +1770,7 @@ impl ComputerUseLinux { "type_text", Err(error.message), received, - focus, + kde_focus, )); } } @@ -4707,6 +4728,7 @@ fn ydotool_type_timeout(text: &str) -> Duration { } const EVDEV_KEY_LEFTCTRL: i32 = 29; +const EVDEV_KEY_LEFTSHIFT: i32 = 42; const EVDEV_KEY_V: i32 = 47; const KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS: u64 = 1_500; const KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS: u64 = 5_000; @@ -4758,6 +4780,7 @@ impl KdeClipboardPasteError { async fn run_kde_clipboard_paste_text( session: &PortalKeyboardSession, text: &str, + use_terminal_paste: bool, operation_guard: InputOperationGuard, ) -> std::result::Result { let previous = kde_clipboard_contents() @@ -4778,7 +4801,7 @@ async fn run_kde_clipboard_paste_text( let paste_result = press_keycode_chord( session, - &[EVDEV_KEY_LEFTCTRL], + kde_clipboard_paste_modifiers(use_terminal_paste), EVDEV_KEY_V, Some(operation_guard), ) @@ -4800,6 +4823,38 @@ async fn run_kde_clipboard_paste_text( } } +async fn kde_clipboard_uses_terminal_paste( + target: &WindowTarget, + focus: Option<&WindowFocusResult>, +) -> bool { + if let Some(focus) = focus { + let window = focus + .focused_window + .as_ref() + .unwrap_or(&focus.requested_window); + return kde_clipboard_target_is_terminal(target, Some(window)); + } + if let Ok(Some(current)) = focused_window().await { + return kde_clipboard_target_is_terminal(target, Some(¤t)); + } + kde_clipboard_target_is_terminal(target, None) +} + +fn kde_clipboard_target_is_terminal(target: &WindowTarget, window: Option<&WindowInfo>) -> bool { + match window { + Some(window) => uses_terminal_paste_shortcut(window), + None => target.has_terminal_target(), + } +} + +fn kde_clipboard_paste_modifiers(use_terminal_paste: bool) -> &'static [i32] { + if use_terminal_paste { + &[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT] + } else { + &[EVDEV_KEY_LEFTCTRL] + } +} + async fn kde_clipboard_contents() -> std::result::Result { let connection = kde_clipboard_connection().await?; let proxy = kde_clipboard_proxy(&connection).await?; @@ -6108,6 +6163,103 @@ mod tests { ); } + #[test] + fn kde_clipboard_uses_terminal_paste_shortcut_for_terminals() { + assert_eq!( + kde_clipboard_paste_modifiers(true), + &[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT] + ); + } + + #[test] + fn kde_clipboard_keeps_standard_paste_shortcut_for_other_apps() { + assert_eq!(kde_clipboard_paste_modifiers(false), &[EVDEV_KEY_LEFTCTRL]); + } + + #[test] + fn kde_clipboard_routes_explicit_tty_targets_to_terminal_paste() { + let target = WindowTarget { + tty: Some("/dev/pts/11".to_string()), + ..Default::default() + }; + + assert!(kde_clipboard_target_is_terminal(&target, None)); + } + + #[test] + fn kde_clipboard_prefers_resolved_window_over_stale_terminal_selector() { + let target = WindowTarget { + tty: Some("/dev/pts/11".to_string()), + ..Default::default() + }; + let window = window_info( + 1, + Some("Browser"), + Some("firefox"), + Some("firefox"), + Some(100), + ); + + assert!(!kde_clipboard_target_is_terminal(&target, Some(&window))); + } + + #[tokio::test] + async fn kde_clipboard_async_routing_prefers_focus_over_stale_terminal_selector() { + let target = WindowTarget { + tty: Some("/dev/pts/11".to_string()), + ..Default::default() + }; + let window = window_info( + 1, + Some("Browser"), + Some("firefox"), + Some("firefox"), + Some(100), + ); + let focus = WindowFocusResult { + requested_window: window.clone(), + focused_window: Some(window), + exact_window_focused: true, + app_focused: true, + backend: KWIN_BACKEND.to_string(), + note: "test".to_string(), + }; + + assert!(!kde_clipboard_uses_terminal_paste(&target, Some(&focus)).await); + } + + #[test] + fn kde_clipboard_routes_konsole_identity_to_terminal_paste() { + let window = window_info( + 1, + Some("unflappable-donkey"), + Some("org.kde.konsole"), + Some("konsole"), + Some(100), + ); + + assert!(kde_clipboard_target_is_terminal( + &WindowTarget::default(), + Some(&window) + )); + } + + #[test] + fn kde_clipboard_does_not_classify_terminal_words_in_browser_titles() { + let window = window_info( + 1, + Some("xterm documentation"), + Some("firefox"), + Some("firefox"), + Some(100), + ); + + assert!(!kde_clipboard_target_is_terminal( + &WindowTarget::default(), + Some(&window) + )); + } + #[test] fn ambiguous_kde_clipboard_set_never_allows_input_replay() { let error = KdeClipboardPasteError::ambiguous_clipboard_set("timed out".to_string()); diff --git a/computer-use-linux/src/terminal.rs b/computer-use-linux/src/terminal.rs index 02d6eb5f3..63e9d96c4 100644 --- a/computer-use-linux/src/terminal.rs +++ b/computer-use-linux/src/terminal.rs @@ -216,34 +216,78 @@ fn process_depth(pid: u32, ancestor_pid: u32, by_pid: &HashMap bool { - let haystack = [ - window.app_id.as_deref(), - window.wm_class.as_deref(), - window.title.as_deref(), - ] - .into_iter() - .flatten() - .collect::>() - .join(" ") - .to_ascii_lowercase(); - - [ - "ghostty", - "gnome-terminal", - "org.gnome.terminal", - "ptyxis", - "org.gnome.ptyxis", - "kgx", - "konsole", - "kitty", - "alacritty", - "wezterm", - "xterm", - ] - .iter() - .any(|needle| haystack.contains(needle)) + uses_terminal_paste_shortcut(window) + || window.title.as_deref().is_some_and(|title| { + let title = title.to_ascii_lowercase(); + TERMINAL_TITLE_HINTS + .iter() + .any(|needle| title.contains(needle)) + }) } +pub(crate) fn uses_terminal_paste_shortcut(window: &WindowInfo) -> bool { + window.terminal.is_some() + || [window.app_id.as_deref(), window.wm_class.as_deref()] + .into_iter() + .flatten() + .any(terminal_identity_matches) +} + +fn terminal_identity_matches(value: &str) -> bool { + let identity = value.trim().to_ascii_lowercase(); + let identity = identity.strip_suffix(".desktop").unwrap_or(&identity); + TERMINAL_IDENTITIES.contains(&identity) +} + +const TERMINAL_IDENTITIES: &[&str] = &[ + "alacritty", + "com.gexperts.tilix", + "com.mitchellh.ghostty", + "com.system76.cosmicterm", + "foot", + "gnome-terminal", + "gnome-terminal-server", + "io.elementary.terminal", + "kitty", + "kgx", + "konsole", + "lxterminal", + "mate-terminal", + "org.codeberg.dnkl.foot", + "org.gnome.console", + "org.gnome.ptyxis", + "org.gnome.terminal", + "org.kde.konsole", + "org.kde.yakuake", + "org.lxqt.qterminal", + "org.wezfurlong.wezterm", + "ptyxis", + "qterminal", + "rxvt", + "rxvt-unicode", + "sakura", + "terminator", + "tilix", + "urxvt", + "uxterm", + "wezterm", + "wezterm-gui", + "xfce4-terminal", + "xterm", + "yakuake", +]; + +const TERMINAL_TITLE_HINTS: &[&str] = &[ + "alacritty", + "ghostty", + "gnome terminal", + "konsole", + "kitty", + "ptyxis", + "wezterm", + "xterm", +]; + fn read_process_table() -> Vec { let Ok(entries) = fs::read_dir("/proc") else { return Vec::new(); @@ -419,6 +463,60 @@ mod tests { assert!(windows.iter().all(|window| window.terminal.is_none())); } + #[test] + fn terminal_paste_identity_matching_is_exact() { + let mut window = terminal_window(11, 100); + window.app_id = Some("com.example.footnotes".to_string()); + window.wm_class = Some("kitty-helper".to_string()); + + assert!(!uses_terminal_paste_shortcut(&window)); + } + + #[test] + fn terminal_paste_recognizes_common_terminal_identities() { + for identity in [ + "org.kde.konsole", + "org.gnome.Terminal", + "org.gnome.Ptyxis", + "kgx", + "uxterm", + "xfce4-terminal", + "org.codeberg.dnkl.foot.desktop", + ] { + let mut window = terminal_window(11, 100); + window.app_id = Some(identity.to_string()); + window.wm_class = None; + assert!( + uses_terminal_paste_shortcut(&window), + "did not recognize {identity}" + ); + } + } + + #[test] + fn terminal_paste_accepts_wm_class_or_enriched_pty_metadata() { + let mut window = terminal_window(11, 100); + window.app_id = None; + window.wm_class = Some("qterminal".to_string()); + assert!(uses_terminal_paste_shortcut(&window)); + + window.wm_class = None; + window.terminal = Some(TerminalWindowContext { + tty: "/dev/pts/11".to_string(), + root_process: TerminalProcess { + pid: 200, + command_name: "bash".to_string(), + command_line: "bash".to_string(), + cwd: Some("/home/user".to_string()), + }, + active_process: None, + process_count: 1, + confidence: "high".to_string(), + match_reason: "test".to_string(), + }); + assert!(uses_terminal_paste_shortcut(&window)); + } + #[test] fn parses_proc_stat_with_parenthesized_command() { let stat =