diff --git a/src/Cameras/computer_vision/computer_vision/cv2_utils.py b/src/Cameras/computer_vision/computer_vision/cv2_utils.py new file mode 100644 index 00000000..e535e2d2 --- /dev/null +++ b/src/Cameras/computer_vision/computer_vision/cv2_utils.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +""" +OpenCV ArUco Compatibility Layer + +This module provides a compatibility layer for OpenCV ArUco API differences +between versions 4.5.4 and 4.12.0. It detects the OpenCV version and uses +the appropriate API calls. + +Key API changes between versions: +- cv2.aruco.Dictionary_get() -> cv2.aruco.getPredefinedDictionary() +- cv2.aruco.DetectorParameters_create() -> cv2.aruco.DetectorParameters() +- cv2.aruco.Board_create() -> cv2.aruco.Board() +- cv2.aruco.detectMarkers() signature changes +- cv2.aruco.drawDetectedMarkers() signature changes +- cv2.aruco.estimatePoseBoard() signature changes +- cv2.aruco.drawAxis() -> cv2.drawFrameAxes() + +Author: GitHub Copilot +Version: 2025-12-07 +""" + +import cv2 +import numpy as np +from packaging import version + + +def get_opencv_version(): + """Get the OpenCV version as a comparable version object""" + return version.parse(cv2.__version__) + + +# Version breakpoint: 4.7.0 is when the API changed significantly +OPENCV_VERSION = get_opencv_version() +IS_NEW_API = OPENCV_VERSION >= version.parse("4.7.0") + +# Mapping from string names to OpenCV dictionary constants (case-insensitive). +_DICT_NAME_MAP = { + "dict_4x4_50": cv2.aruco.DICT_4X4_50, + "dict_4x4_100": cv2.aruco.DICT_4X4_100, + "dict_4x4_250": cv2.aruco.DICT_4X4_250, + "dict_4x4_1000": cv2.aruco.DICT_4X4_1000, + "dict_5x5_50": cv2.aruco.DICT_5X5_50, + "dict_5x5_100": cv2.aruco.DICT_5X5_100, + "dict_5x5_250": cv2.aruco.DICT_5X5_250, + "dict_5x5_1000": cv2.aruco.DICT_5X5_1000, + "dict_6x6_50": cv2.aruco.DICT_6X6_50, + "dict_6x6_100": cv2.aruco.DICT_6X6_100, + "dict_6x6_250": cv2.aruco.DICT_6X6_250, + "dict_6x6_1000": cv2.aruco.DICT_6X6_1000, + "dict_7x7_50": cv2.aruco.DICT_7X7_50, + "dict_7x7_100": cv2.aruco.DICT_7X7_100, + "dict_7x7_250": cv2.aruco.DICT_7X7_250, + "dict_7x7_1000": cv2.aruco.DICT_7X7_1000, + "dict_aruco_original": cv2.aruco.DICT_ARUCO_ORIGINAL, +} + + +def _normalize_dictionary_id(dictionary_id): + if isinstance(dictionary_id, str): + key = dictionary_id.strip().lower() + if key not in _DICT_NAME_MAP: + raise ValueError( + f"Unknown ArUco dictionary name '{dictionary_id}'. " + f"Valid options: {sorted(_DICT_NAME_MAP.keys())}" + ) + return _DICT_NAME_MAP[key] + return dictionary_id + + +def get_aruco_dictionary(dictionary_id): + """ + Get an ArUco dictionary compatible with the current OpenCV version. + + Args: + dictionary_id: Integer constant for the dictionary (e.g., cv2.aruco.DICT_6X6_250) + + Returns: + ArUco dictionary object + """ + dictionary_id = _normalize_dictionary_id(dictionary_id) + if IS_NEW_API: + return cv2.aruco.getPredefinedDictionary(dictionary_id) + else: + return cv2.aruco.Dictionary_get(dictionary_id) + + +def create_detector_parameters(): + """ + Create ArUco detector parameters compatible with the current OpenCV version. + + Returns: + ArUco DetectorParameters object + """ + if IS_NEW_API: + return cv2.aruco.DetectorParameters() + else: + return cv2.aruco.DetectorParameters_create() + + +def create_aruco_board(obj_points, dictionary, ids): + """ + Create an ArUco board compatible with the current OpenCV version. + + Args: + obj_points: Array of 3D object points for each marker + dictionary: ArUco dictionary object + ids: Array of marker IDs + + Returns: + ArUco Board object + """ + if IS_NEW_API: + # In new API, Board constructor takes objPoints and dictionary and ids + return cv2.aruco.Board(obj_points, dictionary, ids) + else: + return cv2.aruco.Board_create(obj_points, dictionary, ids) + + +def detect_markers(image, dictionary, parameters): + """ + Detect ArUco markers in an image compatible with the current OpenCV version. + + Args: + image: Input image + dictionary: ArUco dictionary object + parameters: ArUco detector parameters + + Returns: + Tuple of (corners, ids, rejected) where: + - corners: List of detected marker corners + - ids: Array of detected marker IDs + - rejected: List of rejected marker candidates + """ + if IS_NEW_API: + detector = cv2.aruco.ArucoDetector(dictionary, parameters) + corners, ids, rejected = detector.detectMarkers(image) + else: + corners, ids, rejected = cv2.aruco.detectMarkers( + image, dictionary, parameters=parameters + ) + + return corners, ids, rejected + + +def draw_detected_markers(image, corners, ids): + """ + Draw detected ArUco markers on an image compatible with the current OpenCV version. + + Args: + image: Input/output image + corners: List of detected marker corners + ids: Array of detected marker IDs + + Returns: + Image with markers drawn (modifies in-place and returns) + """ + return cv2.aruco.drawDetectedMarkers(image, corners, ids) + + +def estimate_pose_board(corners, ids, board, camera_matrix, dist_coeffs): + """ + Estimate the pose of an ArUco board compatible with the current OpenCV version. + + Args: + corners: List of detected marker corners + ids: Array of detected marker IDs + board: ArUco Board object + camera_matrix: Camera intrinsic matrix + dist_coeffs: Camera distortion coefficients + + Returns: + Tuple of (num_markers, rvec, tvec) where: + - num_markers: Number of markers used for pose estimation + - rvec: Rotation vector + - tvec: Translation vector + """ + if IS_NEW_API: + # New API: returns (rvec, tvec) directly, and success is determined by non-None values + # The signature is: estimatePoseBoard(corners, ids, board, cameraMatrix, distCoeffs[, rvec[, tvec]]) -> retval, rvec, tvec + obj_points, img_points = board.matchImagePoints(corners, ids) + + if obj_points is None or len(obj_points) == 0: + return 0, None, None + + success, rvec, tvec = cv2.solvePnP( + obj_points, + img_points, + camera_matrix, + dist_coeffs, + flags=cv2.SOLVEPNP_ITERATIVE, + ) + + if success: + return ( + len(obj_points) // 4, + rvec, + tvec, + ) # Divide by 4 because 4 corners per marker + else: + return 0, None, None + else: + # Old API + num_markers, rvec, tvec = cv2.aruco.estimatePoseBoard( + corners, ids, board, camera_matrix, dist_coeffs, None, None + ) + return num_markers, rvec, tvec + + +def estimate_pose_single_markers( + corners, + marker_size: float, + camera_matrix: np.ndarray, + dist_coeffs: np.ndarray, +): + """Estimate individual poses for each detected ArUco marker. + + Parameters + ---------- + corners: + Output of :func:`detect_markers`. + marker_size: + Physical side length of each marker in metres. + camera_matrix, dist_coeffs: + Camera intrinsics (e.g. from ``CameraInfo``). + + Returns + ------- + rvecs : list of np.ndarray + Rotation vectors (Rodrigues), one per marker, shape ``(1, 1, 3)``. + tvecs : list of np.ndarray + Translation vectors, one per marker, shape ``(1, 1, 3)``. + """ + try: + if hasattr(cv2.aruco, "estimatePoseSingleMarkers"): + rvecs, tvecs, _ = cv2.aruco.estimatePoseSingleMarkers( + corners, marker_size, camera_matrix, dist_coeffs + ) + return rvecs, tvecs + except cv2.error: + # Fall back to a manual solvePnP implementation below. + pass + + return _estimate_pose_single_markers_fallback( + corners, marker_size, camera_matrix, dist_coeffs + ) + + +def _estimate_pose_single_markers_fallback( + corners, + marker_size: float, + camera_matrix: np.ndarray, + dist_coeffs: np.ndarray, +): + """Fallback pose estimator using solvePnP for each detected marker.""" + if corners is None: + return [], [] + + obj_points = _marker_object_points(marker_size) + rvecs = [] + tvecs = [] + + if hasattr(cv2, "SOLVEPNP_IPPE_SQUARE"): + primary_flag = cv2.SOLVEPNP_IPPE_SQUARE + else: + primary_flag = cv2.SOLVEPNP_ITERATIVE + + for corner in corners: + img_points = np.asarray(corner, dtype=np.float32).reshape(-1, 2) + success, rvec, tvec = cv2.solvePnP( + obj_points, + img_points, + camera_matrix, + dist_coeffs, + flags=primary_flag, + ) + if not success and primary_flag != cv2.SOLVEPNP_ITERATIVE: + success, rvec, tvec = cv2.solvePnP( + obj_points, + img_points, + camera_matrix, + dist_coeffs, + flags=cv2.SOLVEPNP_ITERATIVE, + ) + if success: + rvecs.append(rvec.reshape(1, 1, 3)) + tvecs.append(tvec.reshape(1, 1, 3)) + + return rvecs, tvecs + + +def _marker_object_points(marker_size: float) -> np.ndarray: + """Return marker corner points centered at the marker origin.""" + half = marker_size / 2.0 + return np.array( + [ + [-half, half, 0.0], + [half, half, 0.0], + [half, -half, 0.0], + [-half, -half, 0.0], + ], + dtype=np.float32, + ) + + +def average_poses(rvecs, tvecs) -> tuple[np.ndarray, np.ndarray]: + """Average a list of poses into a single representative pose. + + Translations are mean-averaged. Rotations are averaged via SVD projection + onto SO(3) so the result is always a valid rotation matrix. + + Parameters + ---------- + rvecs, tvecs: + Lists/arrays of rotation and translation vectors from + :func:`estimate_pose_single_markers`. + + Returns + ------- + rvec_mean : np.ndarray, shape ``(3, 1)`` + tvec_mean : np.ndarray, shape ``(3, 1)`` + """ + tvec_mean = np.mean([t.flatten() for t in tvecs], axis=0, keepdims=True).reshape( + 3, 1 + ) + + rot_sum = np.zeros((3, 3), dtype=np.float64) + for rv in rvecs: + R, _ = cv2.Rodrigues(rv.flatten()) + rot_sum += R + + U, _, Vt = np.linalg.svd(rot_sum) + R_mean = U @ Vt + if np.linalg.det(R_mean) < 0: # ensure proper rotation (det = +1) + U[:, -1] *= -1 + R_mean = U @ Vt + + rvec_mean, _ = cv2.Rodrigues(R_mean) + return rvec_mean, tvec_mean + + +def draw_axis(image, camera_matrix, dist_coeffs, rvec, tvec, length): + """ + Draw coordinate axes on an image compatible with the current OpenCV version. + + Args: + image: Input/output image + camera_matrix: Camera intrinsic matrix + dist_coeffs: Camera distortion coefficients + rvec: Rotation vector + tvec: Translation vector + length: Length of the axes to draw + + Returns: + Image with axes drawn (modifies in-place and returns) + """ + if IS_NEW_API: + # New API uses cv2.drawFrameAxes + return cv2.drawFrameAxes( + image, camera_matrix, dist_coeffs, rvec, tvec, length, 2 + ) + else: + # Old API uses cv2.aruco.drawAxis + return cv2.aruco.drawAxis(image, camera_matrix, dist_coeffs, rvec, tvec, length) + + +def get_api_info(): + """ + Get information about the OpenCV version and API being used. + + Returns: + Dictionary with version information + """ + return { + "opencv_version": cv2.__version__, + "parsed_version": str(OPENCV_VERSION), + "using_new_api": IS_NEW_API, + "api_version": "4.7.0+" if IS_NEW_API else "< 4.7.0", + } diff --git a/src/Cameras/computer_vision/computer_vision/keyboard_pnp_locator.py b/src/Cameras/computer_vision/computer_vision/keyboard_pnp_locator.py new file mode 100644 index 00000000..f55468c0 --- /dev/null +++ b/src/Cameras/computer_vision/computer_vision/keyboard_pnp_locator.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Keyboard locator using per-marker PnP pose estimation. + +Detects ArUco markers affixed to the keyboard corners and calls +cv2.aruco.estimatePoseSingleMarkers on each visible marker, then averages +the resulting poses to produce a stable keyboard pose in the camera frame. + +This approach requires only the marker size (2 cm per URC 2026 rules) and +does NOT depend on knowing the exact inter-marker distances in advance. + +Camera calibration is loaded from a YAML file at startup so the node works +with the wireless USB arm camera, which does not publish a CameraInfo topic. +The frame is pre-undistorted before ArUco detection to improve corner +detection quality under high radial distortion (k1 ≈ 0.60). + +The resulting pose is broadcast as a TF transform: + parent : camera optical frame (from image header) + child : keyboard_center_link (configurable via 'output_frame' param) +""" + +import rclpy +from rclpy.node import Node +from rclpy.qos import qos_profile_sensor_data + +from sensor_msgs.msg import Image, CameraInfo +from geometry_msgs.msg import TransformStamped +from tf2_ros import TransformBroadcaster +from cv_bridge import CvBridge +from scipy.spatial.transform import Rotation + +import cv2 +import numpy as np +import yaml + +from . import cv2_utils + + +class KeyboardPnPLocator(Node): + """Estimates keyboard pose from ArUco markers via per-marker PnP.""" + + MIN_MARKERS = 2 + + def __init__(self): + super().__init__("keyboard_pnp_locator") + + self.declare_parameter("config_file", "config/keyboard_board_config.yaml") + self.declare_parameter( + "camera_calibration_file", "config/end_effector_calibration.yaml" + ) + self.declare_parameter("output_frame", "keyboard_center_link") + self.declare_parameter("image_topic", "/EndEffector/image_raw") + self.declare_parameter("camera_info_topic", "") # empty = don't subscribe + # URC 2026 navigation posts use DICT_4X4_50; keyboard markers likely the same. + # Override via launch argument if the organizers specify otherwise. + self.declare_parameter("aruco_dict", "DICT_4X4_50") + + config_file = self.get_parameter("config_file").value + calib_file = self.get_parameter("camera_calibration_file").value + self.output_frame = self.get_parameter("output_frame").value + image_topic = self.get_parameter("image_topic").value + camera_info_topic = self.get_parameter("camera_info_topic").value + aruco_dict_name = self.get_parameter("aruco_dict").value + + self.tf_broadcaster = TransformBroadcaster(self) + self.bridge = CvBridge() + + # Undistortion maps — pre-computed once from calibration file. + # After remapping, we use new_camera_matrix with zero distortion. + self._map_x: np.ndarray | None = None + self._map_y: np.ndarray | None = None + self._pose_camera_matrix: np.ndarray | None = ( + None # new K for undistorted frame + ) + + self._aruco_dict = cv2_utils.get_aruco_dictionary(aruco_dict_name) + self._aruco_params = cv2_utils.create_detector_parameters() + self._marker_size, self._valid_ids = self._load_board_config(config_file) + self._load_calibration(calib_file) + + # Optional CameraInfo subscription — only wired up when topic is set. + if camera_info_topic: + self.create_subscription( + CameraInfo, + camera_info_topic, + self._camera_info_callback, + qos_profile_sensor_data, + ) + + self.create_subscription( + Image, + image_topic, + self._image_callback, + qos_profile_sensor_data, + ) + + id_info = self._valid_ids.tolist() if self._valid_ids is not None else "any" + self.get_logger().info( + f"Keyboard PnP locator ready — dict={aruco_dict_name}, " + f"marker_size={self._marker_size * 100:.1f} cm, " + f"valid IDs={id_info} — waiting for images..." + ) + + # ------------------------------------------------------------------ + # Config / Calibration loading + # ------------------------------------------------------------------ + + def _load_board_config(self, config_file: str) -> tuple[float, np.ndarray | None]: + with open(config_file, "r") as f: + cfg = yaml.safe_load(f) + marker_size = float(cfg["marker_size"]) + # `ids` is optional. If omitted, all detected markers are accepted — + # useful when the competition has not yet published the marker IDs. + valid_ids = np.array(cfg["ids"], dtype=np.int32) if "ids" in cfg else None + return marker_size, valid_ids + + def _load_calibration(self, calib_file: str) -> None: + """Load K and D from a YAML calibration file and pre-compute undistortion maps.""" + with open(calib_file, "r") as f: + cfg = yaml.safe_load(f) + + w = int(cfg["image_width"]) + h = int(cfg["image_height"]) + + K = np.array(cfg["camera_matrix"]["data"], dtype=np.float64).reshape(3, 3) + D = np.array(cfg["distortion_coefficients"]["data"], dtype=np.float64) + + # alpha=1 keeps all original pixels so corner markers near edges aren't clipped. + new_K, _ = cv2.getOptimalNewCameraMatrix( + K, D, (w, h), alpha=1, newImgSize=(w, h) + ) + map_x, map_y = cv2.initUndistortRectifyMap( + K, D, None, new_K, (w, h), cv2.CV_32FC1 + ) + + self._map_x = map_x + self._map_y = map_y + self._pose_camera_matrix = new_K + + self.get_logger().info( + f"Loaded calibration from {calib_file} — " + f"image {w}x{h}, D coeffs={len(D)}, undistortion maps ready" + ) + + def _camera_info_callback(self, msg: CameraInfo) -> None: + """Optional: recompute undistortion maps if a CameraInfo topic is available.""" + K = np.array(msg.k, dtype=np.float64).reshape(3, 3) + D = np.array(msg.d, dtype=np.float64) + w, h = msg.width, msg.height + + new_K, _ = cv2.getOptimalNewCameraMatrix( + K, D, (w, h), alpha=1, newImgSize=(w, h) + ) + self._map_x, self._map_y = cv2.initUndistortRectifyMap( + K, D, None, new_K, (w, h), cv2.CV_32FC1 + ) + self._pose_camera_matrix = new_K + + # ------------------------------------------------------------------ + # Callbacks + # ------------------------------------------------------------------ + + def _image_callback(self, msg: Image): + if self._map_x is None: + self.get_logger().warn( + "Calibration not loaded yet, skipping frame", + throttle_duration_sec=2.0, + ) + return + + raw = self.bridge.imgmsg_to_cv2(msg, desired_encoding="bgr8") + + # Pre-undistort the frame — improves ArUco corner detection under + # high radial distortion (k1 ≈ 0.60 for this camera). + undistorted = cv2.remap(raw, self._map_x, self._map_y, cv2.INTER_LINEAR) + + corners, ids, _ = cv2_utils.detect_markers( + undistorted, self._aruco_dict, self._aruco_params + ) + + if ids is None or len(ids) < self.MIN_MARKERS: + self.get_logger().warn( + f"Only {0 if ids is None else len(ids)} marker(s) visible " + f"(need {self.MIN_MARKERS}) — skipping", + throttle_duration_sec=1.0, + ) + return + + # Filter to known IDs when configured; otherwise accept everything. + if self._valid_ids is not None: + mask = np.isin(ids.flatten(), self._valid_ids) + if mask.sum() < self.MIN_MARKERS: + self.get_logger().warn( + f"Only {mask.sum()} known marker(s) visible — skipping", + throttle_duration_sec=1.0, + ) + return + corners = [c for c, m in zip(corners, mask) if m] + ids = ids[mask] + + # Frame is already undistorted → pass new_K and zero distortion. + zero_D = np.zeros(5, dtype=np.float64) + rvecs, tvecs = cv2_utils.estimate_pose_single_markers( + corners, self._marker_size, self._pose_camera_matrix, zero_D + ) + + rvec_mean, tvec_mean = cv2_utils.average_poses(rvecs, tvecs) + + self._broadcast_tf(msg.header, rvec_mean, tvec_mean) + t = tvec_mean.flatten() + self.get_logger().info( + f"Keyboard pose [{len(ids)} markers] — " + f"pos=({t[0]:.3f}, {t[1]:.3f}, {t[2]:.3f}) m", + throttle_duration_sec=0.5, + ) + + # ------------------------------------------------------------------ + # TF broadcast + # ------------------------------------------------------------------ + + def _broadcast_tf(self, header, rvec: np.ndarray, tvec: np.ndarray): + rot_mat, _ = cv2.Rodrigues(rvec) + quat = Rotation.from_matrix(rot_mat).as_quat() # [x, y, z, w] + t = tvec.flatten() + + tf_msg = TransformStamped() + tf_msg.header = header + tf_msg.child_frame_id = self.output_frame + + tf_msg.transform.translation.x = float(t[0]) + tf_msg.transform.translation.y = float(t[1]) + tf_msg.transform.translation.z = float(t[2]) + tf_msg.transform.rotation.x = float(quat[0]) + tf_msg.transform.rotation.y = float(quat[1]) + tf_msg.transform.rotation.z = float(quat[2]) + tf_msg.transform.rotation.w = float(quat[3]) + + self.tf_broadcaster.sendTransform(tf_msg) + + +def main(args=None): + rclpy.init(args=args) + node = KeyboardPnPLocator() + try: + rclpy.spin(node) + except KeyboardInterrupt: + node.get_logger().info("Shutting down...") + finally: + node.destroy_node() + if rclpy.ok(): + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/src/Cameras/computer_vision/computer_vision/nav_marker_locator.py b/src/Cameras/computer_vision/computer_vision/nav_marker_locator.py new file mode 100644 index 00000000..788648c0 --- /dev/null +++ b/src/Cameras/computer_vision/computer_vision/nav_marker_locator.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Navigation marker locator using per-marker PnP pose estimation. + +Detects ArUco navigation markers in an image stream, estimates a pose for + +each marker via cv2.aruco.estimatePoseSingleMarkers (or fallback), and +publishes the marker centers as an ArucoMarkers message plus a PoseArray +for visualization. + +This node mirrors the keyboard PnP locator design for calibration loading +and image handling, while using the ZED ArUco detector's publication style. +""" + +import rclpy +from rclpy.node import Node +from rclpy.qos import qos_profile_sensor_data + +from sensor_msgs.msg import Image, CameraInfo +from geometry_msgs.msg import Point, PoseArray, Pose +from interfaces.msg import ArucoMarkers +from cv_bridge import CvBridge + +import cv2 +import numpy as np +import yaml + +from . import cv2_utils + + +class NavMarkerLocator(Node): + """Publishes detected navigation markers as 3D points in camera frame.""" + + def __init__(self): + super().__init__("nav_marker_locator") + + self.declare_parameter("config_file", "config/nav_marker_config.yaml") + self.declare_parameter( + "camera_calibration_file", "config/drive_calibration.yaml" + ) + self.declare_parameter("image_topic", "/Drive/image_raw") + self.declare_parameter("camera_info_topic", "") # empty = don't subscribe + self.declare_parameter("aruco_dict", "DICT_4X4_50") + + config_file = self.get_parameter("config_file").value + calib_file = self.get_parameter("camera_calibration_file").value + image_topic = self.get_parameter("image_topic").value + camera_info_topic = self.get_parameter("camera_info_topic").value + aruco_dict_name = self.get_parameter("aruco_dict").value + + self.bridge = CvBridge() + self._aruco_dict = cv2_utils.get_aruco_dictionary(aruco_dict_name) + self._aruco_params = cv2_utils.create_detector_parameters() + + self._map_x: np.ndarray | None = None + self._map_y: np.ndarray | None = None + self._pose_camera_matrix: np.ndarray | None = None + self._marker_size, self._valid_ids = self._load_board_config(config_file) + self._load_calibration(calib_file) + + if camera_info_topic: + self.create_subscription( + CameraInfo, + camera_info_topic, + self._camera_info_callback, + qos_profile_sensor_data, + ) + + self.create_subscription( + Image, + image_topic, + self._image_callback, + qos_profile_sensor_data, + ) + + self.marker_pub = self.create_publisher( + ArucoMarkers, "/computer_vision/nav_markers", 10 + ) + self.marker_pose_pub = self.create_publisher( + PoseArray, "/computer_vision/nav_marker_poses_viz", 10 + ) + + id_info = self._valid_ids.tolist() if self._valid_ids is not None else "any" + self.get_logger().info( + f"Nav marker locator ready — dict={aruco_dict_name}, " + f"marker_size={self._marker_size * 100:.1f} cm, valid IDs={id_info}" + ) + + # ------------------------------------------------------------------ + # Config / Calibration loading + # ------------------------------------------------------------------ + + def _load_board_config(self, config_file: str) -> tuple[float, np.ndarray | None]: + with open(config_file, "r") as f: + cfg = yaml.safe_load(f) + marker_size = float(cfg["marker_size"]) + valid_ids = np.array(cfg["ids"], dtype=np.int32) if "ids" in cfg else None + return marker_size, valid_ids + + def _load_calibration(self, calib_file: str) -> None: + with open(calib_file, "r") as f: + cfg = yaml.safe_load(f) + + w = int(cfg["image_width"]) + h = int(cfg["image_height"]) + + K = np.array(cfg["camera_matrix"]["data"], dtype=np.float64).reshape(3, 3) + D = np.array(cfg["distortion_coefficients"]["data"], dtype=np.float64) + + new_K, _ = cv2.getOptimalNewCameraMatrix( + K, D, (w, h), alpha=1, newImgSize=(w, h) + ) + map_x, map_y = cv2.initUndistortRectifyMap( + K, D, None, new_K, (w, h), cv2.CV_32FC1 + ) + + self._map_x = map_x + self._map_y = map_y + self._pose_camera_matrix = new_K + + self.get_logger().info( + f"Loaded calibration from {calib_file} — image {w}x{h}, " + f"D coeffs={len(D)}, undistortion maps ready" + ) + + def _camera_info_callback(self, msg: CameraInfo) -> None: + K = np.array(msg.k, dtype=np.float64).reshape(3, 3) + D = np.array(msg.d, dtype=np.float64) + w, h = msg.width, msg.height + + new_K, _ = cv2.getOptimalNewCameraMatrix( + K, D, (w, h), alpha=1, newImgSize=(w, h) + ) + self._map_x, self._map_y = cv2.initUndistortRectifyMap( + K, D, None, new_K, (w, h), cv2.CV_32FC1 + ) + self._pose_camera_matrix = new_K + + # ------------------------------------------------------------------ + # Callbacks + # ------------------------------------------------------------------ + + def _image_callback(self, msg: Image) -> None: + if self._map_x is None: + self.get_logger().warn( + "Calibration not loaded yet, skipping frame", + throttle_duration_sec=2.0, + ) + return + + raw = self.bridge.imgmsg_to_cv2(msg, desired_encoding="bgr8") + undistorted = cv2.remap(raw, self._map_x, self._map_y, cv2.INTER_LINEAR) + + corners, ids, _ = cv2_utils.detect_markers( + undistorted, self._aruco_dict, self._aruco_params + ) + + marker_msg = ArucoMarkers() + marker_msg.header = msg.header + + pose_array = PoseArray() + pose_array.header = msg.header + + if ids is None or len(ids) == 0: + self.marker_pub.publish(marker_msg) + self.marker_pose_pub.publish(pose_array) + return + + if self._valid_ids is not None: + mask = np.isin(ids.flatten(), self._valid_ids) + if not mask.any(): + self.marker_pub.publish(marker_msg) + self.marker_pose_pub.publish(pose_array) + return + corners = [c for c, m in zip(corners, mask) if m] + ids = ids[mask] + + zero_D = np.zeros(5, dtype=np.float64) + rvecs, tvecs = cv2_utils.estimate_pose_single_markers( + corners, self._marker_size, self._pose_camera_matrix, zero_D + ) + + for marker_id, tvec in zip(ids.flatten(), tvecs): + t = tvec.flatten() + marker_msg.marker_ids.append(int(marker_id)) + marker_msg.points.append(Point(x=float(t[0]), y=float(t[1]), z=float(t[2]))) + marker_msg.is_moving.append(False) + + pose = Pose() + pose.position.x = float(t[0]) + pose.position.y = float(t[1]) + pose.position.z = float(t[2]) + pose.orientation.w = 1.0 + pose_array.poses.append(pose) + + self.marker_pub.publish(marker_msg) + self.marker_pose_pub.publish(pose_array) + + +def main(args=None): + rclpy.init(args=args) + node = NavMarkerLocator() + try: + rclpy.spin(node) + except KeyboardInterrupt: + node.get_logger().info("Shutting down...") + finally: + node.destroy_node() + if rclpy.ok(): + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/src/Cameras/computer_vision/computer_vision/zed_aruco_detector.py b/src/Cameras/computer_vision/computer_vision/zed_aruco_detector.py index 42c28a5e..77c9933d 100644 --- a/src/Cameras/computer_vision/computer_vision/zed_aruco_detector.py +++ b/src/Cameras/computer_vision/computer_vision/zed_aruco_detector.py @@ -6,9 +6,10 @@ from interfaces.msg import ArucoMarkers from geometry_msgs.msg import Point, PoseArray, Pose -import cv2 import numpy as np +from . import cv2_utils + class ZEDArucoDetector(Node): def __init__(self): @@ -23,8 +24,8 @@ def __init__(self): self.fy = 0 # Focal length in y-direction self.cx = 0 # Principal point x-coordinate self.cy = 0 # Principal point y-coordinate - self._aruco_detector_params = cv2.aruco.DetectorParameters_create() - self._aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250) + self._aruco_detector_params = cv2_utils.create_detector_parameters() + self._aruco_dict = cv2_utils.get_aruco_dictionary("DICT_4X4_250") # Subscribe to the ZED colour depth image topic self.create_subscription( @@ -81,8 +82,8 @@ def process_image(self): colour_image = self.colour_image[:, :, :3] # Detect ArUco markers in the image - corners, ids, _ = cv2.aruco.detectMarkers( - colour_image, self._aruco_dict, parameters=self._aruco_detector_params + corners, ids, _ = cv2_utils.detect_markers( + colour_image, self._aruco_dict, self._aruco_detector_params ) # Create an ArucoMarkers message diff --git a/src/Cameras/computer_vision/config/drive_calibration.yaml b/src/Cameras/computer_vision/config/drive_calibration.yaml new file mode 100644 index 00000000..7a1b1081 --- /dev/null +++ b/src/Cameras/computer_vision/config/drive_calibration.yaml @@ -0,0 +1,22 @@ +# Camera calibration for camera-0 (wireless USB arm camera) +# Lens model: LENSMODEL_OPENCV8 / rational_polynomial +# Image size: 720 x 576 +# +# D = [k1, k2, p1, p2, k3, k4, k5, k6] (8-coefficient rational polynomial) +# k1 = 0.60 is high distortion — frame must be pre-undistorted before detection. + +image_width: 720 +image_height: 576 + +camera_matrix: + rows: 3 + cols: 3 + data: [234.270200500, 0.000000000, 346.423740500, + 0.000000000, 255.869438100, 287.528427900, + 0.000000000, 0.000000000, 1.000000000] + +distortion_coefficients: + rows: 1 + cols: 8 + data: [ 6.030760110e-01, 4.734409150e-01, -1.139866073e-03, -4.822030303e-04, + -7.081831399e-02, 3.530105362e-01, 6.553218691e-01, -7.595088572e-02] diff --git a/src/Cameras/computer_vision/config/end_effector_calibration.yaml b/src/Cameras/computer_vision/config/end_effector_calibration.yaml new file mode 100644 index 00000000..7a1b1081 --- /dev/null +++ b/src/Cameras/computer_vision/config/end_effector_calibration.yaml @@ -0,0 +1,22 @@ +# Camera calibration for camera-0 (wireless USB arm camera) +# Lens model: LENSMODEL_OPENCV8 / rational_polynomial +# Image size: 720 x 576 +# +# D = [k1, k2, p1, p2, k3, k4, k5, k6] (8-coefficient rational polynomial) +# k1 = 0.60 is high distortion — frame must be pre-undistorted before detection. + +image_width: 720 +image_height: 576 + +camera_matrix: + rows: 3 + cols: 3 + data: [234.270200500, 0.000000000, 346.423740500, + 0.000000000, 255.869438100, 287.528427900, + 0.000000000, 0.000000000, 1.000000000] + +distortion_coefficients: + rows: 1 + cols: 8 + data: [ 6.030760110e-01, 4.734409150e-01, -1.139866073e-03, -4.822030303e-04, + -7.081831399e-02, 3.530105362e-01, 6.553218691e-01, -7.595088572e-02] diff --git a/src/Cameras/computer_vision/config/keyboard_board_config.yaml b/src/Cameras/computer_vision/config/keyboard_board_config.yaml new file mode 100644 index 00000000..42681013 --- /dev/null +++ b/src/Cameras/computer_vision/config/keyboard_board_config.yaml @@ -0,0 +1,13 @@ +# Keyboard ArUco Marker Configuration — URC 2026 +# +# Only marker_size is required. ids is optional: +# - Omit ids (or comment it out) to accept ANY detected marker. +# - Set ids once the competition publishes which IDs will be used. +# +# Per URC 2026 rules: markers are 2 x 2 cm squares placed at keyboard corners. + +# Printed side length of each ArUco marker (metres). Fixed by URC 2026 rules. +marker_size: 0.020 + +# Marker IDs — uncomment and set once the organizers publish them. +# ids: [0, 1, 2, 3] diff --git a/src/Cameras/computer_vision/config/keyboard_key_layout.yaml b/src/Cameras/computer_vision/config/keyboard_key_layout.yaml new file mode 100644 index 00000000..5c22da80 --- /dev/null +++ b/src/Cameras/computer_vision/config/keyboard_key_layout.yaml @@ -0,0 +1,347 @@ +# Redragon K552 ANSI TKL key layout — auto-generated by generate_key_layout.py +# Key positions are keycap centres in the keyboard frame (metres). +# Origin: top-left corner of keyboard surface. +X right, +Y down, +Z out. +# +# To update border offsets: edit BORDER_LEFT / BORDER_TOP in +# scripts/generate_key_layout.py and re-run the script. +# +key_unit_m: 0.01905 +border_left_m: 0.0105 +border_top_m: 0.0155 +keys: + esc: + - 0.02003 + - 0.0155 + - 0.0 + f1: + - 0.05813 + - 0.0155 + - 0.0 + f2: + - 0.07717 + - 0.0155 + - 0.0 + f3: + - 0.09623 + - 0.0155 + - 0.0 + f4: + - 0.11528 + - 0.0155 + - 0.0 + f5: + - 0.14385 + - 0.0155 + - 0.0 + f6: + - 0.1629 + - 0.0155 + - 0.0 + f7: + - 0.18195 + - 0.0155 + - 0.0 + f8: + - 0.201 + - 0.0155 + - 0.0 + f9: + - 0.22958 + - 0.0155 + - 0.0 + f10: + - 0.24863 + - 0.0155 + - 0.0 + f11: + - 0.26767 + - 0.0155 + - 0.0 + f12: + - 0.28673 + - 0.0155 + - 0.0 + grave: + - 0.02003 + - 0.04408 + - 0.0 + '1': + - 0.03908 + - 0.04408 + - 0.0 + '2': + - 0.05813 + - 0.04408 + - 0.0 + '3': + - 0.07717 + - 0.04408 + - 0.0 + '4': + - 0.09623 + - 0.04408 + - 0.0 + '5': + - 0.11528 + - 0.04408 + - 0.0 + '6': + - 0.13432 + - 0.04408 + - 0.0 + '7': + - 0.15338 + - 0.04408 + - 0.0 + '8': + - 0.17243 + - 0.04408 + - 0.0 + '9': + - 0.19148 + - 0.04408 + - 0.0 + '0': + - 0.21053 + - 0.04408 + - 0.0 + minus: + - 0.22958 + - 0.04408 + - 0.0 + equal: + - 0.24863 + - 0.04408 + - 0.0 + backspace: + - 0.2772 + - 0.04408 + - 0.0 + tab: + - 0.02479 + - 0.06313 + - 0.0 + q: + - 0.0486 + - 0.06313 + - 0.0 + w: + - 0.06765 + - 0.06313 + - 0.0 + e: + - 0.0867 + - 0.06313 + - 0.0 + r: + - 0.10575 + - 0.06313 + - 0.0 + t: + - 0.1248 + - 0.06313 + - 0.0 + y: + - 0.14385 + - 0.06313 + - 0.0 + u: + - 0.1629 + - 0.06313 + - 0.0 + i: + - 0.18195 + - 0.06313 + - 0.0 + o: + - 0.201 + - 0.06313 + - 0.0 + p: + - 0.22005 + - 0.06313 + - 0.0 + lbracket: + - 0.2391 + - 0.06313 + - 0.0 + rbracket: + - 0.25815 + - 0.06313 + - 0.0 + backslash: + - 0.28196 + - 0.06313 + - 0.0 + capslock: + - 0.02717 + - 0.08217 + - 0.0 + a: + - 0.05336 + - 0.08217 + - 0.0 + s: + - 0.07241 + - 0.08217 + - 0.0 + d: + - 0.09146 + - 0.08217 + - 0.0 + f: + - 0.11051 + - 0.08217 + - 0.0 + g: + - 0.12956 + - 0.08217 + - 0.0 + h: + - 0.14861 + - 0.08217 + - 0.0 + j: + - 0.16766 + - 0.08217 + - 0.0 + k: + - 0.18671 + - 0.08217 + - 0.0 + l: + - 0.20576 + - 0.08217 + - 0.0 + semicolon: + - 0.22481 + - 0.08217 + - 0.0 + quote: + - 0.24386 + - 0.08217 + - 0.0 + enter: + - 0.27482 + - 0.08217 + - 0.0 + lshift: + - 0.03193 + - 0.10123 + - 0.0 + z: + - 0.06289 + - 0.10123 + - 0.0 + x: + - 0.08194 + - 0.10123 + - 0.0 + c: + - 0.10099 + - 0.10123 + - 0.0 + v: + - 0.12004 + - 0.10123 + - 0.0 + b: + - 0.13909 + - 0.10123 + - 0.0 + n: + - 0.15814 + - 0.10123 + - 0.0 + m: + - 0.17719 + - 0.10123 + - 0.0 + comma: + - 0.19624 + - 0.10123 + - 0.0 + period: + - 0.21529 + - 0.10123 + - 0.0 + slash: + - 0.23434 + - 0.10123 + - 0.0 + rshift: + - 0.27006 + - 0.10123 + - 0.0 + lctrl: + - 0.02241 + - 0.12028 + - 0.0 + win: + - 0.04622 + - 0.12028 + - 0.0 + lalt: + - 0.07003 + - 0.12028 + - 0.0 + space: + - 0.14147 + - 0.12028 + - 0.0 + ralt: + - 0.21291 + - 0.12028 + - 0.0 + fn_key: + - 0.23672 + - 0.12028 + - 0.0 + menu: + - 0.26053 + - 0.12028 + - 0.0 + rctrl: + - 0.28434 + - 0.12028 + - 0.0 + insert: + - 0.3153 + - 0.04408 + - 0.0 + home: + - 0.33435 + - 0.04408 + - 0.0 + pageup: + - 0.3534 + - 0.04408 + - 0.0 + delete: + - 0.3153 + - 0.06313 + - 0.0 + end: + - 0.33435 + - 0.06313 + - 0.0 + pagedown: + - 0.3534 + - 0.06313 + - 0.0 + up: + - 0.33435 + - 0.10123 + - 0.0 + left: + - 0.3153 + - 0.12028 + - 0.0 + down: + - 0.33435 + - 0.12028 + - 0.0 + right: + - 0.3534 + - 0.12028 + - 0.0 diff --git a/src/Cameras/computer_vision/config/nav_marker_config.yaml b/src/Cameras/computer_vision/config/nav_marker_config.yaml new file mode 100644 index 00000000..548f2616 --- /dev/null +++ b/src/Cameras/computer_vision/config/nav_marker_config.yaml @@ -0,0 +1,4 @@ +# Navigation marker configuration +# marker_size is in metres +marker_size: 0.15 +# ids: [1, 2, 3, 4] diff --git a/src/Cameras/computer_vision/launch/keyboard_pnp_locator.launch.py b/src/Cameras/computer_vision/launch/keyboard_pnp_locator.launch.py new file mode 100644 index 00000000..5a3bb343 --- /dev/null +++ b/src/Cameras/computer_vision/launch/keyboard_pnp_locator.launch.py @@ -0,0 +1,53 @@ +from launch import LaunchDescription +from launch.actions import DeclareLaunchArgument +from launch.substitutions import LaunchConfiguration +from launch_ros.actions import Node +from launch_ros.substitutions import FindPackageShare +from launch.substitutions import PathJoinSubstitution + + +def generate_launch_description(): + pkg = FindPackageShare("computer_vision") + + config_file = PathJoinSubstitution([pkg, "config", "keyboard_board_config.yaml"]) + calib_file = PathJoinSubstitution([pkg, "config", "camera_calibration.yaml"]) + + return LaunchDescription( + [ + DeclareLaunchArgument( + "image_topic", + default_value="/EndEffector/image_raw", + description="Raw image topic from the arm camera", + ), + DeclareLaunchArgument( + "camera_info_topic", + default_value="", + description="CameraInfo topic (leave empty to use calibration file)", + ), + DeclareLaunchArgument( + "output_frame", + default_value="keyboard_center_link", + ), + DeclareLaunchArgument( + "aruco_dict", + default_value="DICT_4X4_50", + description="ArUco dictionary name (e.g. DICT_4X4_50, DICT_4X4_100)", + ), + Node( + package="computer_vision", + executable="keyboard_pnp_locator_node", + name="keyboard_pnp_locator_node", + output="screen", + parameters=[ + { + "config_file": config_file, + "camera_calibration_file": calib_file, + "image_topic": LaunchConfiguration("image_topic"), + "camera_info_topic": LaunchConfiguration("camera_info_topic"), + "output_frame": LaunchConfiguration("output_frame"), + "aruco_dict": LaunchConfiguration("aruco_dict"), + } + ], + ), + ] + ) diff --git a/src/Cameras/computer_vision/scripts/generate_key_layout.py b/src/Cameras/computer_vision/scripts/generate_key_layout.py new file mode 100644 index 00000000..2306e1f6 --- /dev/null +++ b/src/Cameras/computer_vision/scripts/generate_key_layout.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Generate keyboard_key_layout.yaml for the Redragon K552 (ANSI TKL, 87-key). + +Key positions are keycap centres in the keyboard frame: + origin: top-left corner of keyboard surface + +X right, +Y down, +Z out of surface (toward camera) + +Usage: + python3 generate_key_layout.py + +IMPORTANT: Measure BORDER_LEFT and BORDER_TOP on the physical keyboard +and update the values below before competition. +""" + +import os +import yaml + +KEY_UNIT = 0.01905 # 1U = 19.05 mm + +# Distance from keyboard surface edge to the centre of the nearest key. +# Measure on the physical keyboard and update these two values. +BORDER_LEFT = 0.0105 # metres ← MEASURE AND UPDATE +BORDER_TOP = 0.0155 # metres ← MEASURE AND UPDATE (fn row centre from top edge) + +# Row Y positions in key units from the fn row centre. +# The K552 has a ~0.5U gap between the fn row and the number row. +ROW_Y = { + "fn": 0.0, + "num": 1.5, + "qwerty": 2.5, + "asdf": 3.5, + "zxcv": 4.5, + "bottom": 5.5, +} + +NAV_X = 15.5 # X offset (units) where nav/arrow cluster begins + + +def _key(name, x_left_u, width_u, row): + x_centre = x_left_u + width_u / 2.0 + return (name, x_centre, ROW_Y[row]) + + +def _build_layout(): + keys = [] + + # Function row + keys += [ + _key("esc", 0.0, 1.0, "fn"), + _key("f1", 2.0, 1.0, "fn"), + _key("f2", 3.0, 1.0, "fn"), + _key("f3", 4.0, 1.0, "fn"), + _key("f4", 5.0, 1.0, "fn"), + _key("f5", 6.5, 1.0, "fn"), + _key("f6", 7.5, 1.0, "fn"), + _key("f7", 8.5, 1.0, "fn"), + _key("f8", 9.5, 1.0, "fn"), + _key("f9", 11.0, 1.0, "fn"), + _key("f10", 12.0, 1.0, "fn"), + _key("f11", 13.0, 1.0, "fn"), + _key("f12", 14.0, 1.0, "fn"), + ] + + # Number row + x = 0.0 + for name in [ + "grave", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "minus", + "equal", + ]: + keys.append(_key(name, x, 1.0, "num")) + x += 1.0 + keys.append(_key("backspace", x, 2.0, "num")) + + # QWERTY row + x = 0.0 + keys.append(_key("tab", x, 1.5, "qwerty")) + x += 1.5 + for name in [ + "q", + "w", + "e", + "r", + "t", + "y", + "u", + "i", + "o", + "p", + "lbracket", + "rbracket", + ]: + keys.append(_key(name, x, 1.0, "qwerty")) + x += 1.0 + keys.append(_key("backslash", x, 1.5, "qwerty")) + + # ASDF row + x = 0.0 + keys.append(_key("capslock", x, 1.75, "asdf")) + x += 1.75 + for name in ["a", "s", "d", "f", "g", "h", "j", "k", "l", "semicolon", "quote"]: + keys.append(_key(name, x, 1.0, "asdf")) + x += 1.0 + keys.append(_key("enter", x, 2.25, "asdf")) + + # ZXCV row + x = 0.0 + keys.append(_key("lshift", x, 2.25, "zxcv")) + x += 2.25 + for name in ["z", "x", "c", "v", "b", "n", "m", "comma", "period", "slash"]: + keys.append(_key(name, x, 1.0, "zxcv")) + x += 1.0 + keys.append(_key("rshift", x, 2.75, "zxcv")) + + # Bottom row + keys += [ + _key("lctrl", 0.0, 1.25, "bottom"), + _key("win", 1.25, 1.25, "bottom"), + _key("lalt", 2.5, 1.25, "bottom"), + _key("space", 3.75, 6.25, "bottom"), + _key("ralt", 10.0, 1.25, "bottom"), + _key("fn_key", 11.25, 1.25, "bottom"), + _key("menu", 12.5, 1.25, "bottom"), + _key("rctrl", 13.75, 1.25, "bottom"), + ] + + # Navigation cluster + keys += [ + _key("insert", NAV_X + 0.0, 1.0, "num"), + _key("home", NAV_X + 1.0, 1.0, "num"), + _key("pageup", NAV_X + 2.0, 1.0, "num"), + _key("delete", NAV_X + 0.0, 1.0, "qwerty"), + _key("end", NAV_X + 1.0, 1.0, "qwerty"), + _key("pagedown", NAV_X + 2.0, 1.0, "qwerty"), + _key("up", NAV_X + 1.0, 1.0, "zxcv"), + _key("left", NAV_X + 0.0, 1.0, "bottom"), + _key("down", NAV_X + 1.0, 1.0, "bottom"), + _key("right", NAV_X + 2.0, 1.0, "bottom"), + ] + + return keys + + +def generate(out_path=None): + layout = _build_layout() + + keys_dict = {} + for name, x_u, y_u in layout: + x_m = round(BORDER_LEFT + x_u * KEY_UNIT, 5) + y_m = round(BORDER_TOP + y_u * KEY_UNIT, 5) + keys_dict[name] = [x_m, y_m, 0.0] + + config = { + "key_unit_m": KEY_UNIT, + "border_left_m": BORDER_LEFT, + "border_top_m": BORDER_TOP, + "keys": keys_dict, + } + + if out_path is None: + out_path = os.path.normpath( + os.path.join( + os.path.dirname(__file__), "..", "config", "keyboard_key_layout.yaml" + ) + ) + + with open(out_path, "w") as f: + f.write( + "# Redragon K552 ANSI TKL key layout — auto-generated by generate_key_layout.py\n" + "# Key positions are keycap centres in the keyboard frame (metres).\n" + "# Origin: top-left corner of keyboard surface. +X right, +Y down, +Z out.\n" + "#\n" + "# To update border offsets: edit BORDER_LEFT / BORDER_TOP in\n" + "# scripts/generate_key_layout.py and re-run the script.\n" + "#\n" + ) + yaml.dump( + config, f, default_flow_style=False, allow_unicode=True, sort_keys=False + ) + + print(f"Written: {out_path}") + print(f"Total keys: {len(keys_dict)}") + return out_path + + +if __name__ == "__main__": + generate() diff --git a/src/Cameras/computer_vision/setup.py b/src/Cameras/computer_vision/setup.py index 079a8b71..c4df2c60 100644 --- a/src/Cameras/computer_vision/setup.py +++ b/src/Cameras/computer_vision/setup.py @@ -10,7 +10,8 @@ data_files=[ ("share/ament_index/resource_index/packages", ["resource/" + package_name]), ("share/" + package_name, ["package.xml"]), - ("share/" + package_name, glob("launch/*.py")), + ("share/" + package_name + "/launch", glob("launch/*.py")), + ("share/" + package_name + "/config", glob("config/*.yaml")), ], install_requires=["setuptools"], zip_safe=True, @@ -21,7 +22,9 @@ tests_require=["pytest"], entry_points={ "console_scripts": [ - "zed_aruco_detector_node = computer_vision.zed_aruco_detector:main" + "zed_aruco_detector_node = computer_vision.zed_aruco_detector:main", + "keyboard_pnp_locator_node = computer_vision.keyboard_pnp_locator:main", + "nav_marker_locator_node = computer_vision.nav_marker_locator:main", ], }, ) diff --git a/src/Cameras/video_streaming/CMakeLists.txt b/src/Cameras/video_streaming/CMakeLists.txt index ffc67e18..3eff6b71 100644 --- a/src/Cameras/video_streaming/CMakeLists.txt +++ b/src/Cameras/video_streaming/CMakeLists.txt @@ -12,6 +12,7 @@ find_package(rclcpp REQUIRED) find_package(rclcpp_components REQUIRED) find_package(interfaces REQUIRED) find_package(std_msgs REQUIRED) +find_package(sensor_msgs REQUIRED) find_package(PkgConfig REQUIRED) pkg_check_modules(GSTREAMER REQUIRED IMPORTED_TARGET gstreamer-1.0) @@ -61,6 +62,7 @@ ament_target_dependencies(${PROJECT_NAME} rclcpp_components interfaces std_msgs + sensor_msgs ) # If pkg-config provided extra compile flags (e.g., -D, -pthread) diff --git a/src/Cameras/video_streaming/include/video_streaming/detect_node.hpp b/src/Cameras/video_streaming/include/video_streaming/detect_node.hpp index 7d52b20f..0a01b3e4 100644 --- a/src/Cameras/video_streaming/include/video_streaming/detect_node.hpp +++ b/src/Cameras/video_streaming/include/video_streaming/detect_node.hpp @@ -1,7 +1,10 @@ #pragma once #include "base_video_node.hpp" +#include #include +#include #include +#include class DetectNode : public BaseVideoNode { public: @@ -22,8 +25,13 @@ class DetectNode : public BaseVideoNode { rclcpp::Publisher::SharedPtr marker_pub_; rclcpp::Publisher::SharedPtr object_detected_pub_; + rclcpp::Subscription::SharedPtr active_camera_sub_; + rclcpp::Publisher::SharedPtr arm_cam_pub_; + std::string active_camera_{"Drive"}; + void active_camera_callback(const std_msgs::msg::String::SharedPtr msg); rcl_interfaces::msg::SetParametersResult on_parameter_change(const std::vector ¶meters); DetectionType string_to_detection_type(const std::string &type_str); std::string detection_type_to_string() const; + static GstFlowReturn on_new_sample(GstAppSink *sink, gpointer user_data); }; diff --git a/src/Cameras/video_streaming/include/video_streaming/input_node.hpp b/src/Cameras/video_streaming/include/video_streaming/input_node.hpp index 1b21dcc8..cc8e48ed 100644 --- a/src/Cameras/video_streaming/include/video_streaming/input_node.hpp +++ b/src/Cameras/video_streaming/include/video_streaming/input_node.hpp @@ -2,6 +2,7 @@ #include "base_video_node.hpp" #include #include +#include class InputNode : public BaseVideoNode { public: @@ -32,5 +33,6 @@ class InputNode : public BaseVideoNode { std::map source_map_; rclcpp::Service::SharedPtr video_service_; rclcpp::Service::SharedPtr get_cam_service_; + rclcpp::Publisher::SharedPtr active_camera_pub_; std::shared_ptr current_video_request_; }; diff --git a/src/Cameras/video_streaming/package.xml b/src/Cameras/video_streaming/package.xml index 84955559..ffa9888c 100644 --- a/src/Cameras/video_streaming/package.xml +++ b/src/Cameras/video_streaming/package.xml @@ -13,6 +13,7 @@ rclcpp rclcpp_components std_msgs + sensor_msgs gstreamer-1.0 glib-2.0 libssl-dev diff --git a/src/Cameras/video_streaming/src/detect_node.cpp b/src/Cameras/video_streaming/src/detect_node.cpp index a3d851eb..cf00884c 100644 --- a/src/Cameras/video_streaming/src/detect_node.cpp +++ b/src/Cameras/video_streaming/src/detect_node.cpp @@ -21,6 +21,12 @@ DetectNode::DetectNode(const rclcpp::NodeOptions &options) this->declare_parameter("rockpick_config", "config/rockpick/rockpick.txt"); this->declare_parameter("listen_to", "input"); + + active_camera_sub_ = this->create_subscription( + "/active_camera", 10, + std::bind(&DetectNode::active_camera_callback, this, + std::placeholders::_1)); + param_callback_handle_ = this->add_on_set_parameters_callback( std::bind(&DetectNode::on_parameter_change, this, std::placeholders::_1)); marker_pub_ = this->create_publisher( @@ -49,6 +55,7 @@ bool DetectNode::create_pipeline() { "allow-renegotiation=true name=src ! "; detection_type_ = string_to_detection_type( this->get_parameter("detection_type").as_string()); + bool add_terminator = true; switch (detection_type_) { case DetectionType::WATER_BOTTLE: desc_ss << get_detection_pipeline_str( @@ -63,15 +70,26 @@ bool DetectNode::create_pipeline() { this->get_parameter("rockpick_config").as_string()); break; case DetectionType::ARUCO: - desc_ss << "videoconvert ! queue ! videoconvert ! arucomarker " - "detect-every=10 " - "name=aruco_detector ! queue ! videoconvert ! "; + // Split stream: one branch to interpipesink (for downstream streaming), + // another to appsink so the Python PnP locator can read frames over ROS + // without opening the USB device directly (which would conflict with + // GStreamer). + desc_ss << "videoconvert ! queue ! videoconvert ! " + "arucomarker detect-every=10 name=aruco_detector ! " + "tee name=aruco_tee " + "aruco_tee. ! queue ! videoconvert ! nvvidconv ! interpipesink " + "name=detect " + "aruco_tee. ! queue ! " + "appsink name=arm_cam_sink sync=false max-buffers=1 drop=true"; + add_terminator = false; break; case DetectionType::NONE: desc_ss << "identity ! "; break; } - desc_ss << "nvvidconv ! interpipesink name=detect"; + if (add_terminator) { + desc_ss << "nvvidconv ! interpipesink name=detect"; + } RCLCPP_INFO(this->get_logger(), "Creating pipeline: %s", desc_ss.str().c_str()); @@ -113,6 +131,19 @@ bool DetectNode::start_pipeline() { g_signal_connect(aruco, "marker-detected", G_CALLBACK(on_marker_detected), marker_pub_.get()); gst_object_unref(aruco); + + arm_cam_pub_ = this->create_publisher( + active_camera_ + "/image_raw", rclcpp::QoS(1).best_effort()); + GstElement *appsink = get_element("arm_cam_sink"); + if (!appsink) { + RCLCPP_ERROR(this->get_logger(), "Failed to get arm_cam_sink element."); + return false; + } + GstAppSinkCallbacks callbacks = {}; + callbacks.new_sample = on_new_sample; + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, this, + nullptr); + gst_object_unref(appsink); } else if (detection_type_ == DetectionType::MALLET || detection_type_ == DetectionType::WATER_BOTTLE) { GstElement *osd = get_element("osd"); @@ -220,6 +251,19 @@ std::string DetectNode::detection_type_to_string() const { } } +void DetectNode::active_camera_callback( + const std_msgs::msg::String::SharedPtr msg) { + if (msg->data != active_camera_) { + active_camera_ = msg->data; + RCLCPP_INFO(this->get_logger(), "Active camera changed to %s", + active_camera_.c_str()); + if (arm_cam_pub_) { + arm_cam_pub_ = this->create_publisher( + active_camera_ + "/image_raw", rclcpp::QoS(1).best_effort()); + } + } +} + void DetectNode::publish_object_detected(int32_t class_id, float confidence, int32_t xmin, int32_t ymin, int32_t xmax, int32_t ymax) { @@ -274,4 +318,52 @@ rcl_interfaces::msg::SetParametersResult DetectNode::on_parameter_change( return result; } +GstFlowReturn DetectNode::on_new_sample(GstAppSink *sink, gpointer user_data) { + auto *self = static_cast(user_data); + if (!self->arm_cam_pub_) { + return GST_FLOW_OK; + } + + GstSample *sample = gst_app_sink_pull_sample(sink); + if (!sample) { + return GST_FLOW_ERROR; + } + + GstCaps *caps = gst_sample_get_caps(sample); + GstBuffer *buffer = gst_sample_get_buffer(sample); + + gint width = 0, height = 0; + if (caps) { + GstStructure *s = gst_caps_get_structure(caps, 0); + gst_structure_get_int(s, "width", &width); + gst_structure_get_int(s, "height", &height); + } + + GstMapInfo map; + if (!gst_buffer_map(buffer, &map, GST_MAP_READ)) { + gst_sample_unref(sample); + return GST_FLOW_ERROR; + } + + sensor_msgs::msg::Image msg; + msg.header.stamp = self->now(); + + if (self->active_camera_ == "EndEffector") { + msg.header.frame_id = "EndEffector"; + } else { + msg.header.frame_id = "DriveCamera"; + } + msg.height = static_cast(height); + msg.width = static_cast(width); + msg.encoding = "rgb8"; + msg.is_bigendian = 0; + msg.step = static_cast(width * 3); + msg.data.assign(map.data, map.data + map.size); + self->arm_cam_pub_->publish(std::move(msg)); + + gst_buffer_unmap(buffer, &map); + gst_sample_unref(sample); + return GST_FLOW_OK; +} + RCLCPP_COMPONENTS_REGISTER_NODE(DetectNode) diff --git a/src/Cameras/video_streaming/src/input_node.cpp b/src/Cameras/video_streaming/src/input_node.cpp index c9377b98..b2988dd0 100644 --- a/src/Cameras/video_streaming/src/input_node.cpp +++ b/src/Cameras/video_streaming/src/input_node.cpp @@ -6,6 +6,8 @@ InputNode::InputNode(const rclcpp::NodeOptions &options) : BaseVideoNode("input_node", options) { RCLCPP_INFO(this->get_logger(), "InputNode constructed."); declare_parameters(); + active_camera_pub_ = + this->create_publisher("/active_camera", 10); video_service_ = this->create_service( "start_video", std::bind(&InputNode::video_cb, this, std::placeholders::_1, std::placeholders::_2)); @@ -212,6 +214,11 @@ void InputNode::video_cb( } if (response->success) { current_video_request_ = request; + if (!request->sources.empty()) { + std_msgs::msg::String msg; + msg.data = request->sources[0].name; + active_camera_pub_->publish(msg); + } } }