Add OpenCV lens distortion rendering to Newton

source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst

@@ -0,0 +1,11 @@
+Added
+^^^^^
+
+* Added OpenCV lens-distortion support to the Newton renderer: a camera cfg carrying an OpenCV
+  pinhole (``k1..k6``, ``p1``, ``p2``, ``s1..s4``) or fisheye (``k1..k4``) distortion model on
+  ``spawn.distortion`` is now rendered through the distortion instead of as a centered, square-pixel
+  pinhole. Fisheye cameras use Newton's native OpenCV fisheye ray helper, while pinhole cameras use
+  an Isaac Lab kernel supporting rational radial, tangential, and thin-prism distortion. Both paths
+  honor the calibrated ``fx/fy/cx/cy`` intrinsics (including non-square focal lengths and an
+  off-center principal point). With ``apply_lens_distortion=False`` the distortion coefficients are
+  muted while the intrinsics are still applied, matching the RTX/OVRTX behavior.

source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import logging
+import math
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, NoReturn
 
@@ -145,6 +146,9 @@ def __init__(
         else:
             self.clear_color = 0xFFEEEEEE
 
+        # OpenCV lens-distortion model (``spawn.distortion``), consumed by :meth:`_build_distortion_rays`
+        # to trace distorted per-pixel rays instead of the centered, square-pixel pinhole field.
+        self._distortion = getattr(spawn, "distortion", None)
         # Post-render PPISP pipeline composed when ``spec.cfg.isp_cfg`` is set.
         # ``isp_cfg`` is already fully normalized by ``prepare_cameras`` by the time it reaches here.
         self.ppisp_pipeline: PpispPipeline | None = None
@@ -356,14 +360,86 @@ def update(self, positions: ProxyArray, orientations: ProxyArray, intrinsics: Pr
         )
 
         if self.camera_rays is None:
-            first_focal_length = intrinsics.torch[:, 1, 1][0:1]
-            fov_radians_all = 2.0 * torch.atan(self.height / (2.0 * first_focal_length))
+            if self._distortion is not None:
+                self.camera_rays = self._build_distortion_rays()
+            else:
+                first_focal_length = intrinsics.torch[:, 1, 1][0:1]
+                fov_radians_all = 2.0 * torch.atan(self.height / (2.0 * first_focal_length))
+
+                fov_warp = wp.from_torch(fov_radians_all, dtype=wp.float32)
+                self.camera_rays = self.newton_sensor.utils.compute_camera_rays_pinhole(
+                    self.width, self.height, camera_fovs=fov_warp
+                )
 
-            fov_warp = wp.from_torch(fov_radians_all, dtype=wp.float32)
-            self.camera_rays = self.newton_sensor.utils.compute_camera_rays_pinhole(
-                self.width, self.height, camera_fovs=fov_warp
+    def _build_distortion_rays(self) -> wp.array(dtype=wp.vec3f, ndim=4):
+        """Build the ``(1, H, W, 2)`` camera-space ray field for an OpenCV lens-distortion camera.
+
+        Uses Newton's native OpenCV fisheye ray helper and the Isaac Lab OpenCV pinhole kernel. Both
+        paths honor calibrated ``fx/fy/cx/cy`` (non-square, off-center) intrinsics. When
+        :attr:`OpenCvDistortionCfg.apply_lens_distortion` is ``False``, the coefficients are treated
+        as zero while the calibrated intrinsics remain active, matching the RTX/OVRTX behavior.
+        """
+        from .opencv_distortion_rays import compute_camera_rays_opencv_pinhole
+
+        cfg = self._distortion
+        device = self.newton_sensor.model.device
+        image_width, image_height = float(cfg.image_size[0]), float(cfg.image_size[1])
+        # ``apply_lens_distortion=False`` keeps the intrinsics but mutes the distortion coefficients.
+        apply = bool(getattr(cfg, "apply_lens_distortion", True))
+
+        def _coeff(name: str) -> float:
+            return float(getattr(cfg, name, 0.0)) if apply else 0.0
+
+        if cfg.model == "opencvFisheye":
+            return self.newton_sensor.utils.compute_camera_rays_fisheye_opencv(
+                self.width,
+                self.height,
+                float(cfg.fx),
+                float(cfg.fy),
+                float(cfg.cx),
+                float(cfg.cy),
+                image_width=image_width,
+                image_height=image_height,
+                k1=_coeff("k1"),
+                k2=_coeff("k2"),
+                k3=_coeff("k3"),
+                k4=_coeff("k4"),
+                # Match the PR kernel's forward-facing camera hemisphere and avoid validating the
+                # OpenCV polynomial outside its physically meaningful calibration range.
+                max_fov=math.pi,
             )
 
+        rays = wp.empty((1, self.height, self.width, 2), dtype=wp.vec3f, device=device)
+        wp.launch(
+            compute_camera_rays_opencv_pinhole,
+            dim=(1, self.height, self.width),
+            inputs=[
+                self.width,
+                self.height,
+                float(cfg.fx),
+                float(cfg.fy),
+                float(cfg.cx),
+                float(cfg.cy),
+                image_width,
+                image_height,
+                _coeff("k1"),
+                _coeff("k2"),
+                _coeff("k3"),
+                _coeff("k4"),
+                _coeff("k5"),
+                _coeff("k6"),
+                _coeff("p1"),
+                _coeff("p2"),
+                _coeff("s1"),
+                _coeff("s2"),
+                _coeff("s3"),
+                _coeff("s4"),
+            ],
+            outputs=[rays],
+            device=device,
+        )
+        return rays
+
     @wp.kernel
     def _update_transforms(
         positions: wp.array(dtype=wp.vec3f),
@@ -473,20 +549,12 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None:
 
         Also captures the USD ``stage`` so the segmentation mapper can read the scene's
         :class:`UsdSemantics.LabelsAPI` labels when a segmentation output is requested.
+
+        OpenCV lens distortion (``spawn.distortion``) needs no preparation here: it is consumed at
+        ray-generation time by :meth:`RenderData._build_distortion_rays`, which inverts the OpenCV
+        forward model per pixel to trace the distorted camera-space rays.
         """
         self._stage = stage
-        # NOTE: OpenCV lens distortion (``spawn.distortion``) is not yet applied by the Newton
-        # renderer. The distortion cfg is renderer-agnostic and could be piped through Newton's warp
-        # ray-tracing utilities here in the future; for now the camera renders undistorted. This is
-        # the intended extension point.
-        spawn = getattr(spec.cfg, "spawn", None)
-        if getattr(spawn, "distortion", None) is not None:
-            logger.warning(
-                "OpenCV lens distortion is set on the camera cfg but is not yet applied by the Newton"
-                " renderer: it derives a single field of view from fy, so the distortion coefficients,"
-                " the principal point, and a non-square fx are ignored and the camera renders as a"
-                " centered, square-pixel pinhole. Use the RTX/OVRTX renderer to apply the full model."
-            )
         if spec.cfg.isp_cfg is None:
             return
         try:

source/isaaclab_newton/isaaclab_newton/renderers/opencv_distortion_rays.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Warp ray generation for OpenCV pinhole lens distortion under the Newton renderer.
+
+The Newton tiled camera renders by tracing an explicit per-pixel ray field of shape
+``(camera_count, height, width, 2)`` (``wp.vec3f``): index ``0`` holds the ray origin in camera
+space (always ``wp.vec3f(0.0)``) and index ``1`` the normalized ray direction in camera space.
+Newton uses the OpenGL camera convention (``+X`` right, ``+Y`` up, looking down ``-Z``).
+
+To honor an OpenCV ``fx/fy/cx/cy`` + distortion-coefficient calibration, for each output pixel the
+kernel below inverts the OpenCV forward distortion model to recover the *undistorted* normalized
+image coordinates ``(x_u, y_u)`` (OpenCV image ``y`` points down), then emit the camera-space ray
+``normalize(vec3(x_u, -y_u, -1))`` -- the negation of ``y`` and ``z`` maps OpenCV camera space
+(``+Z`` forward, ``+Y`` down) onto Newton's OpenGL camera space.
+
+OpenCV fisheye ray generation is provided directly by Newton's
+``SensorTiledCamera.utils.compute_camera_rays_fisheye_opencv`` helper.
+"""
+
+from __future__ import annotations
+
+import warp as wp
+
+# Number of fixed-point iterations used to invert the distortion model. OpenCV's own
+# ``undistortPoints`` defaults to a similar iteration budget; the inversion converges well within
+# this for realistic calibrations.
+_INVERSION_ITERATIONS = 20
+
+
+@wp.kernel(enable_backward=False)
+def compute_camera_rays_opencv_pinhole(
+    width: int,
+    height: int,
+    fx: wp.float32,
+    fy: wp.float32,
+    cx: wp.float32,
+    cy: wp.float32,
+    image_width: wp.float32,
+    image_height: wp.float32,
+    k1: wp.float32,
+    k2: wp.float32,
+    k3: wp.float32,
+    k4: wp.float32,
+    k5: wp.float32,
+    k6: wp.float32,
+    p1: wp.float32,
+    p2: wp.float32,
+    s1: wp.float32,
+    s2: wp.float32,
+    s3: wp.float32,
+    s4: wp.float32,
+    out_rays: wp.array(dtype=wp.vec3f, ndim=4),
+):
+    """Emit camera-space rays for an OpenCV pinhole (rational + tangential + thin-prism) camera.
+
+    The forward OpenCV model maps an undistorted normalized point ``(x, y)`` (with ``r2 = x^2 + y^2``)
+    to the distorted normalized point ``(x_d, y_d)`` via a rational radial term, tangential terms
+    (``p1``, ``p2``) and thin-prism terms (``s1..s4``). For each output pixel the distorted point is
+    known from the pixel coordinate and the intrinsics; the kernel recovers the undistorted point by
+    fixed-point iteration (matching OpenCV's :func:`undistortPoints`) and forms the ray.
+
+    Args:
+        width: Output image width [px].
+        height: Output image height [px].
+        fx: Focal length along the image x-axis [px].
+        fy: Focal length along the image y-axis [px].
+        cx: Principal point x-coordinate [px].
+        cy: Principal point y-coordinate [px].
+        image_width: Calibrated image width the intrinsics refer to [px].
+        image_height: Calibrated image height the intrinsics refer to [px].
+        k1: First radial distortion coefficient (numerator).
+        k2: Second radial distortion coefficient (numerator).
+        k3: Third radial distortion coefficient (numerator).
+        k4: First radial distortion coefficient (denominator, rational model).
+        k5: Second radial distortion coefficient (denominator, rational model).
+        k6: Third radial distortion coefficient (denominator, rational model).
+        p1: First tangential distortion coefficient.
+        p2: Second tangential distortion coefficient.
+        s1: First thin-prism distortion coefficient.
+        s2: Second thin-prism distortion coefficient.
+        s3: Third thin-prism distortion coefficient.
+        s4: Fourth thin-prism distortion coefficient.
+        out_rays: Ray field of shape ``(1, height, width, 2)``: ``[..., 0]`` origin, ``[..., 1]``
+            direction, both in Newton's OpenGL camera space.
+    """
+    camera_index, py, px = wp.tid()
+
+    # Map the render pixel onto the calibrated image grid, then to distorted normalized coordinates.
+    # OpenCV image y points down.
+    u = ((wp.float32(px) + 0.5) / wp.float32(width)) * image_width
+    v = ((wp.float32(py) + 0.5) / wp.float32(height)) * image_height
+    x_d = (u - cx) / fx
+    y_d = (v - cy) / fy
+
+    # Fixed-point inversion of the forward model, seeded at the distorted point.
+    x = x_d
+    y = y_d
+    for _i in range(_INVERSION_ITERATIONS):
+        r2 = x * x + y * y
+        r4 = r2 * r2
+        r6 = r4 * r2
+        radial = (1.0 + k1 * r2 + k2 * r4 + k3 * r6) / (1.0 + k4 * r2 + k5 * r4 + k6 * r6)
+        dx = 2.0 * p1 * x * y + p2 * (r2 + 2.0 * x * x) + s1 * r2 + s2 * r4
+        dy = p1 * (r2 + 2.0 * y * y) + 2.0 * p2 * x * y + s3 * r2 + s4 * r4
+        x = (x_d - dx) / radial
+        y = (y_d - dy) / radial
+
+    # OpenCV camera space (+Z forward, +Y down) -> Newton OpenGL camera space (-Z forward, +Y up).
+    ray_direction_camera_space = wp.normalize(wp.vec3f(x, -y, -1.0))
+    out_rays[camera_index, py, px, 0] = wp.vec3f(0.0)
+    out_rays[camera_index, py, px, 1] = ray_direction_camera_space

source/isaaclab_newton/test/renderers/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+"""Tests for Newton renderers."""