diff --git a/README.md b/README.md index 8d3ecda041..173b7bfd93 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,5 @@ In order to keep a clean overview containing all contributed modules, the follow 1. Update the README.md file under the modules folder. Here, you add your model with a single-line description. 2. Add a README.md inside your own module folder. This README explains which functionality (separate functions) is available, links to the corresponding samples, and explains in somewhat more detail what the module is expected to do. If any extra requirements are needed to build the module without problems, add them here also. + +fjut diff --git a/modules/hed/CMakeLists.txt b/modules/hed/CMakeLists.txt new file mode 100644 index 0000000000..b5c7d2d935 --- /dev/null +++ b/modules/hed/CMakeLists.txt @@ -0,0 +1,2 @@ +set(the_module opencv_hed) +ocv_define_module(hed opencv_core opencv_imgproc opencv_dnn WRAP python) \ No newline at end of file diff --git a/modules/hed/README.md b/modules/hed/README.md new file mode 100644 index 0000000000..391f4aaf0b --- /dev/null +++ b/modules/hed/README.md @@ -0,0 +1,26 @@ +# hed — Holistically-nested Edge Detection + +Implements HED edge detection (Xie & Tu, 2015) as an OpenCV contrib module. +Unlike Canny, HED uses a pretrained VGG network to detect semantically +meaningful edges rather than raw pixel contrast changes. + +## Paper +Xie, S., & Tu, 2015. Holistically-nested edge detection. +https://arxiv.org/abs/1504.06375 + +## Model files +Download from: https://vcl.ucsd.edu/hed/ +- hed_pretrained_bsds.caffemodel +- deploy.prototxt + +## Python usage +```python +detector = cv2.hed.HEDDetector.create("model.caffemodel", "deploy.prototxt") +edges = detector.detectEdges(image) # float32, values 0..1 +``` + +## C++ usage +```cpp +auto det = cv::hed::HEDDetector::create("model.caffemodel", "deploy.prototxt"); +cv::Mat edges = det->detectEdges(image); +``` \ No newline at end of file diff --git a/modules/hed/include/opencv2/hed.hpp b/modules/hed/include/opencv2/hed.hpp new file mode 100644 index 0000000000..8df6df5226 --- /dev/null +++ b/modules/hed/include/opencv2/hed.hpp @@ -0,0 +1,52 @@ +#pragma once +#include "opencv2/core.hpp" +#include + +namespace cv { +namespace hed { + +/** @defgroup hed Holistically-nested Edge Detection +This module implements HED edge detection as described in: + +Xie, S., & Tu, Z. (2015). Holistically-nested edge detection. +Proceedings of the IEEE international conference on computer vision. +https://arxiv.org/abs/1504.06375 +*/ + +//! @addtogroup hed +//! @{ + +/** @brief HED edge detector using a pretrained Caffe model. + +Unlike classical detectors such as Canny, HED uses a VGG-based +convolutional network trained on human-annotated boundaries to +produce semantically meaningful edge maps. + +@note Requires hed_pretrained_bsds.caffemodel and deploy.prototxt. +Download from: https://vcl.ucsd.edu/hed/ +*/ +class CV_EXPORTS_W HEDDetector { +public: + /** @brief Creates an HEDDetector instance. + @param modelPath path to hed_pretrained_bsds.caffemodel + @param protoPath path to deploy.prototxt + */ + CV_WRAP static cv::Ptr create( + const std::string& modelPath, + const std::string& protoPath + ); + + /** @brief Detects edges in an image. + @param image input BGR image (CV_8UC3) + @return edge map as CV_32F with values in [0, 1] + */ + CV_WRAP virtual cv::Mat detectEdges(cv::InputArray image) = 0; + + /** @brief Destructor. */ + virtual ~HEDDetector() {} +}; + +//! @} + +} // namespace hed +} // namespace cv \ No newline at end of file diff --git a/modules/hed/samples/camera_background_demo.py b/modules/hed/samples/camera_background_demo.py new file mode 100644 index 0000000000..8a776ca629 --- /dev/null +++ b/modules/hed/samples/camera_background_demo.py @@ -0,0 +1,347 @@ +import cv2 as cv +import numpy as np +import argparse +import time +import os +import sys +import urllib.request + +# Define standard search paths for HED model files +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_PROTO = os.path.join(SCRIPT_DIR, "../../../../opencv/data/deploy.prototxt") +DEFAULT_MODEL = os.path.join(SCRIPT_DIR, "../../../../opencv/data/hed_pretrained_bsds.caffemodel") + +# Fallbacks for absolute paths if running outside the standard build structure +ALT_PROTO = r"n:\dev-stuff\opencvchange\opencv\data\deploy.prototxt" +ALT_MODEL = r"n:\dev-stuff\opencvchange\opencv\data\hed_pretrained_bsds.caffemodel" + +# Parse arguments +parser = argparse.ArgumentParser( + description='Real-time HED vs Canny Virtual Background Replacement Demo. ' + 'Compares edge-based semantic segmentation using HED vs Canny.' +) +parser.add_argument('--input', help='Path to video file or image. Skip to use webcam.', default=0) +parser.add_argument('--prototxt', help='Path to deploy.prototxt', default=None) +parser.add_argument('--caffemodel', help='Path to caffemodel', default=None) +parser.add_argument('--width', help='Inference width', default=384, type=int) +parser.add_argument('--height', help='Inference height', default=384, type=int) +args = parser.parse_args() + +# ------------------------------------------------------------- +# 1. Resolve Model and Background Image Paths +# ------------------------------------------------------------- +proto_path = args.prototxt +model_path = args.caffemodel + +if not proto_path: + if os.path.exists(DEFAULT_PROTO): + proto_path = DEFAULT_PROTO + elif os.path.exists(ALT_PROTO): + proto_path = ALT_PROTO + else: + print(f"Error: Prototxt not found at {DEFAULT_PROTO} or {ALT_PROTO}") + sys.exit(1) + +if not model_path: + if os.path.exists(DEFAULT_MODEL): + model_path = DEFAULT_MODEL + elif os.path.exists(ALT_MODEL): + model_path = ALT_MODEL + else: + print(f"Error: Caffemodel not found at {DEFAULT_MODEL} or {ALT_MODEL}") + sys.exit(1) + +bg_save_path = os.path.join(SCRIPT_DIR, "background.jpg") + +def download_background(save_path): + # Public URLs of beautiful high-res background images (office room/beach) + urls = [ + "https://images.unsplash.com/photo-1497366216548-37526070297c?auto=format&fit=crop&w=800&q=80", # Office Room + "https://images.unsplash.com/photo-1507525428034-b723cf961d3e?auto=format&fit=crop&w=800&q=80" # Beach + ] + if os.path.exists(save_path): + print(f"[INFO] Virtual background image found locally: {save_path}") + return True + + print("[INFO] Downloading virtual background image from Unsplash...") + for url in urls: + try: + # Set a user-agent to prevent HTTP 403 Forbidden from some CDNs + req = urllib.request.Request( + url, + headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} + ) + with urllib.request.urlopen(req) as response, open(save_path, 'wb') as out_file: + out_file.write(response.read()) + print(f"[SUCCESS] Background saved to: {save_path}") + return True + except Exception as e: + print(f"[WARNING] Failed downloading from {url}: {e}") + + print("[WARNING] Could not download background image. Falling back to generated virtual pattern.") + return False + +def generate_fallback_background(h, w): + # Generates a professional vertical gradient background with a grid overlay + background = np.zeros((h, w, 3), dtype=np.uint8) + for y in range(h): + r = 0 + g = int(140 * (y / h)) + b = int(220 * (1.0 - y / h)) + background[y, :, :] = [b, g, r] # BGR format + # Add thin digital gridlines + for x in range(0, w, 40): + cv.line(background, (x, 0), (x, h), (75, 75, 75), 1) + for y in range(0, h, 40): + cv.line(background, (0, y), (w, y), (75, 75, 75), 1) + return background + +# Ensure background image is present +has_bg_file = download_background(bg_save_path) + +# ------------------------------------------------------------- +# 2. Register Custom Caffe Crop Layer (Required for CPU inference) +# ------------------------------------------------------------- +class CropLayer(object): + def __init__(self, params, blobs): + self.xstart = 0 + self.xend = 0 + self.ystart = 0 + self.yend = 0 + + def getMemoryShapes(self, inputs): + inputShape, targetShape = inputs[0], inputs[1] + batchSize, numChannels = inputShape[0], inputShape[1] + height, width = targetShape[2], targetShape[3] + + self.ystart = (inputShape[2] - targetShape[2]) // 2 + self.xstart = (inputShape[3] - targetShape[3]) // 2 + self.yend = self.ystart + height + self.xend = self.xstart + width + + return [[batchSize, numChannels, height, width]] + + def forward(self, inputs): + return [inputs[0][:, :, self.ystart:self.yend, self.xstart:self.xend]] + +# Register standard Caffe Crop Layer +cv.dnn_registerLayer('Crop', CropLayer) + +# ------------------------------------------------------------- +# 3. Load Model and Initialize Video Stream +# ------------------------------------------------------------- +print("[INFO] Loading HED network...") +net = cv.dnn.readNet(proto_path, model_path) +net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV) +net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + +source = args.input +if isinstance(source, str) and source.isdigit(): + source = int(source) + +cap = cv.VideoCapture(source) +if not cap.isOpened(): + print(f"Error: Could not open video source: {args.input}") + sys.exit(1) + +# Default operational parameters +inf_w, inf_h = args.width, args.height +hed_edge_threshold = 0.12 # Lower threshold for better neck/jaw boundary detection +canny_thresh1 = 50 +canny_thresh2 = 120 + +# Window configuration +window_name = "Capstone Demo: Edge-guided Virtual Background replacement (HED vs Canny)" +cv.namedWindow(window_name, cv.WINDOW_NORMAL) +cv.resizeWindow(window_name, 1280, 960) + +print("\n" + "="*50) +print("INTERACTIVE KEYBOARD CONTROLS:") +print(" [q] : Quit demo") +print(" [+ / -] : Adjust HED edge threshold (currently {:.2f})".format(hed_edge_threshold)) +print(" [s] : Save screenshot") +print("="*50 + "\n") + +frame_count = 0 +fps_start_time = time.time() +fps = 0.0 + +# Read initial frame to get sizes and load virtual background +ret, frame = cap.read() +if not ret: + print("[ERROR] Failed to read from camera.") + sys.exit(1) + +h_orig, w_orig = frame.shape[:2] + +if has_bg_file: + bg_img = cv.imread(bg_save_path) + if bg_img is None: + bg_img = generate_fallback_background(h_orig, w_orig) + else: + bg_img = cv.resize(bg_img, (w_orig, h_orig)) +else: + bg_img = generate_fallback_background(h_orig, w_orig) + +while True: + ret, frame = cap.read() + if not ret: + break + + # ------------------------------------------------------------- + # A. Canny Background Separation + # ------------------------------------------------------------- + gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) + canny = cv.Canny(gray, canny_thresh1, canny_thresh2) + + # 1. Close gaps in Canny edges using morphological closing + canny_kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, (25, 25)) + canny_closed = cv.morphologyEx(canny, cv.MORPH_CLOSE, canny_kernel) + canny_dilated = cv.dilate(canny_closed, cv.getStructuringElement(cv.MORPH_ELLIPSE, (3, 3))) + + # 2. Draw border lines ONLY at left, right, and bottom (NOT top) + # This seals the open shoulder boundaries at the borders without enclosing the background + cv.line(canny_dilated, (0, h_orig - 1), (w_orig - 1, h_orig - 1), 255, 5) + cv.line(canny_dilated, (0, 0), (0, h_orig - 1), 255, 5) + cv.line(canny_dilated, (w_orig - 1, 0), (w_orig - 1, h_orig - 1), 255, 5) + + # 3. Find external contours. Since the top is open, the background is not enclosed. + # The only major closed external contour will be the user's silhouette. + canny_contours, _ = cv.findContours(canny_dilated, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + + canny_mask_clean = np.zeros_like(canny) + if canny_contours: + # Keep only the single largest external contour (the user's silhouette) + largest_canny = max(canny_contours, key=cv.contourArea) + # Filling this external contour completely fills the user's interior, ignoring internal facial edges + cv.drawContours(canny_mask_clean, [largest_canny], -1, 255, -1) + + # 4. Smooth mask & blend + canny_mask_smooth = cv.GaussianBlur(canny_mask_clean, (25, 25), 0) / 255.0 + canny_mask_smooth = np.expand_dims(canny_mask_smooth, axis=2) + canny_blended = (frame * canny_mask_smooth + bg_img * (1.0 - canny_mask_smooth)).astype(np.uint8) + + # ------------------------------------------------------------- + # B. HED Background Separation + # ------------------------------------------------------------- + # Create input blob + blob = cv.dnn.blobFromImage( + frame, + scalefactor=1.0, + size=(inf_w, inf_h), + mean=(104.00698793, 116.66876762, 122.67891434), + swapRB=False, + crop=False + ) + net.setInput(blob) + + inf_start = time.time() + hed_out = net.forward() + inf_time = (time.time() - inf_start) * 1000.0 + + # Resize raw probability output back to original camera size + hed_map = hed_out[0, 0] + hed_resized = cv.resize(hed_map, (w_orig, h_orig)) + + # Binarize the probability map to get boundaries + hed_edges = (hed_resized > hed_edge_threshold).astype(np.uint8) * 255 + hed_edges_bgr = cv.cvtColor(hed_edges, cv.COLOR_GRAY2BGR) + + # 1. Close gaps in HED semantic boundaries + hed_kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, (15, 15)) + hed_closed = cv.morphologyEx(hed_edges, cv.MORPH_CLOSE, hed_kernel) + hed_dilated = cv.dilate(hed_closed, cv.getStructuringElement(cv.MORPH_ELLIPSE, (3, 3))) + + # 2. Draw border lines ONLY at left, right, and bottom (NOT top) + # This seals the open shoulder boundaries at the borders without enclosing the background + cv.line(hed_dilated, (0, h_orig - 1), (w_orig - 1, h_orig - 1), 255, 5) + cv.line(hed_dilated, (0, 0), (0, h_orig - 1), 255, 5) + cv.line(hed_dilated, (w_orig - 1, 0), (w_orig - 1, h_orig - 1), 255, 5) + + # 3. Find external contours. Since the top is open, the background is not enclosed. + # The only major closed external contour will be the user's silhouette. + hed_contours, _ = cv.findContours(hed_dilated, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + + hed_mask_clean = np.zeros_like(hed_edges) + if hed_contours: + # Keep only the single largest external contour (the user's silhouette) + largest_hed = max(hed_contours, key=cv.contourArea) + # Filling this external contour completely fills the user's interior (face, shirt, dress), + # ignoring internal facial/clothing texture edges and preventing transparency issues. + cv.drawContours(hed_mask_clean, [largest_hed], -1, 255, -1) + + # 4. Smooth mask & blend + hed_mask_smooth = cv.GaussianBlur(hed_mask_clean, (25, 25), 0) / 255.0 + hed_mask_smooth = np.expand_dims(hed_mask_smooth, axis=2) + hed_blended = (frame * hed_mask_smooth + bg_img * (1.0 - hed_mask_smooth)).astype(np.uint8) + + # ------------------------------------------------------------- + # C. Build 2x2 Grid View + # ------------------------------------------------------------- + font = cv.FONT_HERSHEY_SIMPLEX + text_color = (0, 255, 0) + bg_color = (0, 0, 0) + + # Panel 1: Original image + Stats overlay + p1 = frame.copy() + stats = [ + f"Resolution: {w_orig}x{h_orig}", + f"HED Size: {inf_w}x{inf_h}", + f"HED Latency: {inf_time:.1f} ms", + f"Frame Rate: {fps:.1f} FPS" + ] + for idx, stat in enumerate(stats): + cv.putText(p1, stat, (15, 30 + idx * 25), font, 0.6, bg_color, 4, cv.LINE_AA) + cv.putText(p1, stat, (15, 30 + idx * 25), font, 0.6, text_color, 15, cv.LINE_AA) + cv.putText(p1, stat, (15, 30 + idx * 25), font, 0.6, (255, 255, 255), 1, cv.LINE_AA) + cv.putText(p1, "[1] Original Camera Feed", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # Panel 2: Raw HED Edges + p2 = hed_edges_bgr.copy() + cv.putText(p2, f"[2] HED Semantic Boundaries (Thresh: {hed_edge_threshold:.2f})", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # Panel 3: Canny BG replacement (shows noise and background leaks) + p3 = canny_blended.copy() + cv.putText(p3, "[3] Canny Separation (Noisy Contours)", (15, h_orig - 15), font, 0.7, (0, 0, 255), 2) + + # Panel 4: HED BG replacement (clean semantic contour separation) + p4 = hed_blended.copy() + cv.putText(p4, "[4] HED Separation (Semantic Contours)", (15, h_orig - 15), font, 0.7, (0, 255, 0), 2) + + # Assemble grid + top_row = np.hstack((p1, p2)) + bottom_row = np.hstack((p3, p4)) + grid = np.vstack((top_row, bottom_row)) + + # Scale grid for display + display_scale = 0.75 + h_grid, w_grid = grid.shape[:2] + grid_resized = cv.resize(grid, (int(w_grid * display_scale), int(h_grid * display_scale))) + + cv.imshow(window_name, grid_resized) + + # Calculate FPS + frame_count += 1 + elapsed = time.time() - fps_start_time + if elapsed >= 1.0: + fps = frame_count / elapsed + frame_count = 0 + fps_start_time = time.time() + + # Keyboard interaction + key = cv.waitKey(1) & 0xFF + if key == ord('q'): + break + elif key == ord('+') or key == ord('='): + hed_edge_threshold = min(0.95, hed_edge_threshold + 0.05) + print(f"[ACTION] Increased HED edge binarization threshold to: {hed_edge_threshold:.2f}") + elif key == ord('-') or key == ord('_'): + hed_edge_threshold = max(0.01, hed_edge_threshold - 0.05) + print(f"[ACTION] Decreased HED edge binarization threshold to: {hed_edge_threshold:.2f}") + elif key == ord('s'): + filename = f"bg_separation_comparison_{int(time.time())}.png" + cv.imwrite(filename, grid) + print(f"[ACTION] Saved comparison grid screenshot: {filename}") + +cap.release() +cv.destroyAllWindows() diff --git a/modules/hed/samples/hed_camera_demo.py b/modules/hed/samples/hed_camera_demo.py new file mode 100644 index 0000000000..70996ab510 --- /dev/null +++ b/modules/hed/samples/hed_camera_demo.py @@ -0,0 +1,337 @@ +import cv2 as cv +import numpy as np +import argparse +import time +import os +import sys + +# Define standard search paths for HED model files +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_PROTO = os.path.join(SCRIPT_DIR, "../../../../opencv/data/deploy.prototxt") +DEFAULT_MODEL = os.path.join(SCRIPT_DIR, "../../../../opencv/data/hed_pretrained_bsds.caffemodel") + +# Fallbacks for absolute paths if running outside the standard build structure +ALT_PROTO = r"n:\dev-stuff\opencvchange\opencv\data\deploy.prototxt" +ALT_MODEL = r"n:\dev-stuff\opencvchange\opencv\data\hed_pretrained_bsds.caffemodel" + +# Parse arguments +parser = argparse.ArgumentParser( + description='Real-time Holistically-Nested Edge Detection (HED) webcam demo on CPU. ' + 'Provides interactive GUI controls and comparative visualization.' +) +parser.add_argument('--input', help='Path to video file or image. Skip to use webcam.', default=0) +parser.add_argument('--prototxt', help='Path to deploy.prototxt', default=None) +parser.add_argument('--caffemodel', help='Path to caffemodel', default=None) +parser.add_argument('--width', help='Initial inference width', default=384, type=int) +parser.add_argument('--height', help='Initial inference height', default=384, type=int) +args = parser.parse_args() + +# ------------------------------------------------------------- +# 1. Resolve Model Paths +# ------------------------------------------------------------- +proto_path = args.prototxt +model_path = args.caffemodel + +if not proto_path: + if os.path.exists(DEFAULT_PROTO): + proto_path = DEFAULT_PROTO + elif os.path.exists(ALT_PROTO): + proto_path = ALT_PROTO + else: + print(f"Error: Prototxt not found at {DEFAULT_PROTO} or {ALT_PROTO}") + print("Please specify path using --prototxt") + sys.exit(1) + +if not model_path: + if os.path.exists(DEFAULT_MODEL): + model_path = DEFAULT_MODEL + elif os.path.exists(ALT_MODEL): + model_path = ALT_MODEL + else: + print(f"Error: Caffemodel not found at {DEFAULT_MODEL} or {ALT_MODEL}") + print("Please specify path using --caffemodel") + sys.exit(1) + +print(f"[INFO] Using Prototxt: {proto_path}") +print(f"[INFO] Using Caffemodel: {model_path}") + +# ------------------------------------------------------------- +# 2. Register Custom Caffe Crop Layer (CPU target) +# ------------------------------------------------------------- +class CropLayer(object): + def __init__(self, params, blobs): + self.xstart = 0 + self.xend = 0 + self.ystart = 0 + self.yend = 0 + + def getMemoryShapes(self, inputs): + inputShape, targetShape = inputs[0], inputs[1] + batchSize, numChannels = inputShape[0], inputShape[1] + height, width = targetShape[2], targetShape[3] + + self.ystart = (inputShape[2] - targetShape[2]) // 2 + self.xstart = (inputShape[3] - targetShape[3]) // 2 + self.yend = self.ystart + height + self.xend = self.xstart + width + + return [[batchSize, numChannels, height, width]] + + def forward(self, inputs): + return [inputs[0][:, :, self.ystart:self.yend, self.xstart:self.xend]] + +# Register the crop layer with OpenCV's DNN engine +cv.dnn_registerLayer('Crop', CropLayer) + +# ------------------------------------------------------------- +# 3. Vectorized Non-Maximum Suppression for Edge Thinning +# ------------------------------------------------------------- +def edge_thinning_nms(edge_map, threshold=0.1): + """ + Applies Non-Maximum Suppression (NMS) on a soft edge probability map + by calculating local gradient directions and suppressing non-peak pixels. + """ + # Compute gradients along x and y directions + dx = cv.Sobel(edge_map, cv.CV_32F, 1, 0, ksize=3) + dy = cv.Sobel(edge_map, cv.CV_32F, 0, 1, ksize=3) + + # Compute angles and convert to degrees + angle = np.arctan2(dy, dx) * (180.0 / np.pi) + angle[angle < 0] += 180 + + h, w = edge_map.shape + nms = np.zeros((h, w), dtype=np.float32) + + # Pad image to simplify neighborhood access at boundaries + padded = np.pad(edge_map, 1, mode='constant', constant_values=0) + + # Slices for 8-neighborhood + N = padded[0:-2, 1:-1] + S = padded[2:, 1:-1] + E = padded[1:-1, 2:] + W = padded[1:-1, 0:-2] + NE = padded[0:-2, 2:] + SW = padded[2:, 0:-2] + NW = padded[0:-2, 0:-2] + SE = padded[2:, 2:] + + # Determine the gradient sector + mask0 = ((angle >= 0) & (angle < 22.5)) | ((angle >= 157.5) & (angle <= 180)) + mask45 = (angle >= 22.5) & (angle < 67.5) + mask90 = (angle >= 67.5) & (angle < 112.5) + mask135 = (angle >= 112.5) & (angle < 157.5) + + keep = np.zeros_like(edge_map, dtype=bool) + + # Sector 0 (Horizontal gradient: check East/West) + keep |= mask0 & (edge_map >= W) & (edge_map >= E) + # Sector 1 (45-degree gradient: check NorthWest/SouthEast) + keep |= mask45 & (edge_map >= NW) & (edge_map >= SE) + # Sector 2 (Vertical gradient: check North/South) + keep |= mask90 & (edge_map >= N) & (edge_map >= S) + # Sector 3 (135-degree gradient: check NorthEast/SouthWest) + keep |= mask135 & (edge_map >= NE) & (edge_map >= SW) + + nms[keep] = edge_map[keep] + + # Apply soft thresholding + if threshold > 0: + nms[nms < threshold] = 0 + + return nms + +# ------------------------------------------------------------- +# 4. Initialize Network (CPU mode) +# ------------------------------------------------------------- +print("[INFO] Loading network model (Running on CPU)...") +net = cv.dnn.readNet(proto_path, model_path) +net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV) +net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + +# Start camera / video capture +print("[INFO] Initializing video stream...") +source = args.input +if isinstance(source, str) and source.isdigit(): + source = int(source) + +cap = cv.VideoCapture(source) +if not cap.isOpened(): + print(f"Error: Could not open video source: {args.input}") + sys.exit(1) + +# Default operational parameters +inf_w, inf_h = args.width, args.height +apply_nms = True +nms_threshold = 0.15 +canny_thresh1 = 100 +canny_thresh2 = 200 + +# Window configuration +window_name = "Capstone Demo: HED Real-time comparative analysis (CPU)" +cv.namedWindow(window_name, cv.WINDOW_NORMAL) +cv.resizeWindow(window_name, 1280, 960) + +print("\n" + "="*50) +print("INTERACTIVE KEYBOARD CONTROLS:") +print(" [q] : Quit demo") +print(" [r] : Cycle inference resolution (256 -> 384 -> 500)") +print(" [t] : Toggle Edge Thinning (NMS)") +print(" [+ / -] : Increase/Decrease edge thinning threshold") +print(" [s] : Save a comparison screenshot") +print("="*50 + "\n") + +frame_count = 0 +fps_start_time = time.time() +fps = 0.0 + +while True: + start_time = time.time() + + ret, frame = cap.read() + if not ret: + print("[INFO] End of video stream.") + break + + h_orig, w_orig = frame.shape[:2] + + # ------------------------------------------------------------- + # A. Classic Canny Edge Detection + # ------------------------------------------------------------- + gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) + canny = cv.Canny(gray, canny_thresh1, canny_thresh2) + # Convert Canny output to BGR for display concatenation + canny_bgr = cv.cvtColor(canny, cv.COLOR_GRAY2BGR) + + # ------------------------------------------------------------- + # B. HED Inference via OpenCV DNN (CPU) + # ------------------------------------------------------------- + # Create input blob with VGG BGR training means + blob = cv.dnn.blobFromImage( + frame, + scalefactor=1.0, + size=(inf_w, inf_h), + mean=(104.00698793, 116.66876762, 122.67891434), + swapRB=False, + crop=False + ) + + net.setInput(blob) + + inf_start = time.time() + hed_out = net.forward() + inf_time = (time.time() - inf_start) * 1000.0 # Milliseconds + + # Post-process raw network output (shape: [1, 1, H, W]) + hed_map = hed_out[0, 0] + + # Resize output back to original camera frame resolution + hed_resized = cv.resize(hed_map, (w_orig, h_orig)) + + # Scale from float [0, 1] to uint8 [0, 255] + hed_u8 = (hed_resized * 255).astype(np.uint8) + hed_bgr = cv.cvtColor(hed_u8, cv.COLOR_GRAY2BGR) + + # ------------------------------------------------------------- + # C. HED Edge Thinning (NMS) + # ------------------------------------------------------------- + if apply_nms: + # Perform NMS on the resized float map + thinned = edge_thinning_nms(hed_resized, threshold=nms_threshold) + thinned_u8 = (thinned * 255).astype(np.uint8) + thinned_bgr = cv.cvtColor(thinned_u8, cv.COLOR_GRAY2BGR) + else: + # If disabled, show a blank or bypass image + thinned_bgr = np.zeros_like(frame) + cv.putText(thinned_bgr, "NMS Thinning Disabled", (w_orig // 4, h_orig // 2), + cv.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + # ------------------------------------------------------------- + # D. Overlay Text & Assemble comparative 2x2 Grid + # ------------------------------------------------------------- + font = cv.FONT_HERSHEY_SIMPLEX + text_color = (0, 255, 0) + bg_color = (0, 0, 0) + + # 1. Panel 1: Original Image + Live Stats + orig_annotated = frame.copy() + stats = [ + f"Input: {w_orig}x{h_orig}", + f"Inference Res: {inf_w}x{inf_h}", + f"Active Device: CPU", + f"DNN Latency: {inf_time:.1f} ms", + f"Frame Rate: {fps:.1f} FPS" + ] + for idx, stat in enumerate(stats): + cv.putText(orig_annotated, stat, (15, 30 + idx * 25), font, 0.6, bg_color, 4, cv.LINE_AA) + cv.putText(orig_annotated, stat, (15, 30 + idx * 25), font, 0.6, text_color, 15, cv.LINE_AA) + cv.putText(orig_annotated, stat, (15, 30 + idx * 25), font, 0.6, (255, 255, 255), 1, cv.LINE_AA) + + cv.putText(orig_annotated, "[1] Original Camera Feed", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # 2. Panel 2: Classic Canny + cv.putText(canny_bgr, "[2] Canny Edge Detection", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # 3. Panel 3: Raw HED + cv.putText(hed_bgr, "[3] Raw HED (Probability Map)", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # 4. Panel 4: Thinned HED + cv.putText(thinned_bgr, f"[4] Thinned HED (NMS Thresh: {nms_threshold:.2f})", (15, h_orig - 15), font, 0.7, (0, 255, 255), 2) + + # Construct 2x2 Grid + top_row = np.hstack((orig_annotated, canny_bgr)) + bottom_row = np.hstack((hed_bgr, thinned_bgr)) + grid = np.vstack((top_row, bottom_row)) + + # Scale grid down if it exceeds a typical monitor's resolution + display_scale = 0.75 + h_grid, w_grid = grid.shape[:2] + grid_resized = cv.resize(grid, (int(w_grid * display_scale), int(h_grid * display_scale))) + + cv.imshow(window_name, grid_resized) + + # Calculate global FPS + frame_count += 1 + elapsed = time.time() - fps_start_time + if elapsed >= 1.0: + fps = frame_count / elapsed + frame_count = 0 + fps_start_time = time.time() + + # ------------------------------------------------------------- + # E. Key Handling + # ------------------------------------------------------------- + key = cv.waitKey(1) & 0xFF + if key == ord('q'): + print("[INFO] Exiting...") + break + + elif key == ord('r'): + # Cycle resolutions: 256 -> 384 -> 500 + if inf_w == 256: + inf_w, inf_h = 384, 384 + elif inf_w == 384: + inf_w, inf_h = 500, 500 + else: + inf_w, inf_h = 256, 256 + print(f"[ACTION] Changed network inference size to: {inf_w}x{inf_h}") + + elif key == ord('t'): + apply_nms = not apply_nms + print(f"[ACTION] Toggled NMS Edge Thinning: {apply_nms}") + + elif key == ord('+') or key == ord('='): + nms_threshold = min(0.95, nms_threshold + 0.05) + print(f"[ACTION] Increased NMS threshold to: {nms_threshold:.2f}") + + elif key == ord('-') or key == ord('_'): + nms_threshold = max(0.01, nms_threshold - 0.05) + print(f"[ACTION] Decreased NMS threshold to: {nms_threshold:.2f}") + + elif key == ord('s'): + filename = f"comparison_screenshot_{int(time.time())}.png" + cv.imwrite(filename, grid) + print(f"[ACTION] Saved full screenshot as: {filename}") + +# Cleanup +cap.release() +cv.destroyAllWindows() diff --git a/modules/hed/samples/hed_demo.py b/modules/hed/samples/hed_demo.py new file mode 100644 index 0000000000..85049180b5 --- /dev/null +++ b/modules/hed/samples/hed_demo.py @@ -0,0 +1,28 @@ +import cv2 +import sys +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--image', required=True) +parser.add_argument('--model', required=True) +parser.add_argument('--proto', required=True) +args = parser.parse_args() + +image = cv2.imread(args.image) +if image is None: + print(f"Cannot load image: {args.image}") + sys.exit(1) + +detector = cv2.hed.HEDDetector.create(args.model, args.proto) +edges = detector.detectEdges(image) +edges_u8 = (edges * 255).astype('uint8') + +gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) +canny = cv2.Canny(gray, 100, 200) + +cv2.imshow("Original", image) +cv2.imshow("Canny (classic)", canny) +cv2.imshow("HED (yours)", edges_u8) +cv2.imwrite("hed_result.png", edges_u8) +print("Saved hed_result.png") +cv2.waitKey(0) \ No newline at end of file diff --git a/modules/hed/src/hed.cpp b/modules/hed/src/hed.cpp new file mode 100644 index 0000000000..7906120a2e --- /dev/null +++ b/modules/hed/src/hed.cpp @@ -0,0 +1,43 @@ +#include "opencv2/hed.hpp" +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" + +namespace cv { +namespace hed { + +class HEDDetectorImpl : public HEDDetector { +public: + dnn::Net net; + + HEDDetectorImpl(const std::string& modelPath, const std::string& protoPath) { + net = dnn::readNet(modelPath, protoPath); + } + + cv::Mat detectEdges(cv::InputArray _image) override { + cv::Mat image = _image.getMat(); + cv::Mat blob = dnn::blobFromImage( + image, 1.0, + cv::Size(image.cols, image.rows), + cv::Scalar(104.00698793, 116.66876762, 122.67891434), + false, false + ); + net.setInput(blob); + cv::Mat result = net.forward("sigmoid-fuse"); + + // result shape is [1,1,H,W] — squeeze to [H,W] + cv::Mat edge(result.size[2], result.size[3], CV_32F, result.ptr()); + cv::Mat output; + edge.copyTo(output); + return output; + } +}; + +cv::Ptr HEDDetector::create( + const std::string& modelPath, + const std::string& protoPath) +{ + return cv::makePtr(modelPath, protoPath); +} + +} // hed +} // cv \ No newline at end of file diff --git a/modules/hed/src/precomp.hpp b/modules/hed/src/precomp.hpp new file mode 100644 index 0000000000..117d445904 --- /dev/null +++ b/modules/hed/src/precomp.hpp @@ -0,0 +1,5 @@ +#pragma once +#include "opencv2/core.hpp" +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/hed.hpp" \ No newline at end of file diff --git a/modules/hed/test/test_hed.cpp b/modules/hed/test/test_hed.cpp new file mode 100644 index 0000000000..40fb5ad262 --- /dev/null +++ b/modules/hed/test/test_hed.cpp @@ -0,0 +1,34 @@ +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +TEST(HED, LoadModel) +{ + std::string model = cvtest::findDataFile("hed_pretrained_bsds.caffemodel", false); + std::string proto = cvtest::findDataFile("deploy.prototxt", false); + if (model.empty() || proto.empty()) + throw SkipTestException("HED model files not found"); + + auto detector = cv::hed::HEDDetector::create(model, proto); + ASSERT_FALSE(detector.empty()); +} + +TEST(HED, OutputShape) +{ + std::string model = cvtest::findDataFile("hed_pretrained_bsds.caffemodel", false); + std::string proto = cvtest::findDataFile("deploy.prototxt", false); + if (model.empty() || proto.empty()) + throw SkipTestException("HED model files not found"); + + cv::Mat img = cv::Mat::zeros(100, 100, CV_8UC3); + auto detector = cv::hed::HEDDetector::create(model, proto); + cv::Mat edges = detector->detectEdges(img); + + EXPECT_EQ(edges.rows, 100); + EXPECT_EQ(edges.cols, 100); + EXPECT_EQ(edges.type(), CV_32F); +} + +}} // namespace + +CV_TEST_MAIN(".") \ No newline at end of file diff --git a/modules/hed/test/test_precomp.hpp b/modules/hed/test/test_precomp.hpp new file mode 100644 index 0000000000..8a3b3aecfc --- /dev/null +++ b/modules/hed/test/test_precomp.hpp @@ -0,0 +1,7 @@ +#ifndef __OPENCV_TEST_PRECOMP_HPP__ +#define __OPENCV_TEST_PRECOMP_HPP__ + +#include "opencv2/ts.hpp" +#include "opencv2/hed.hpp" + +#endif \ No newline at end of file