[experiment] ui on the CPU

openpilot/selfdrive/ui/SConscript

@@ -3,6 +3,18 @@ import importlib.util
 
 Import('env', 'arch', 'common')
 
+# Native core for the GPU-free MICI UI backend. It deliberately has no GL/EGL
+# dependencies and is built on Linux hosts so PC parity tests exercise the same
+# code that runs on device.
+if arch != "Darwin":
+  cpu_renderer_env = env.Clone()
+  cpu_renderer_env["CPPPATH"] += ["/usr/include/libdrm"]
+  cpu_renderer_env.SharedLibrary(
+    "#openpilot/system/ui/lib/cpu_renderer/libraylib_cpu",
+    ["#openpilot/system/ui/lib/cpu_renderer/renderer.c"],
+    LIBS=["m", "drm"],
+  )
+
 # build the fonts
 generator = File("#openpilot/selfdrive/assets/fonts/process.py")
 source_files = Glob("#openpilot/selfdrive/assets/fonts/*.ttf") + Glob("#openpilot/selfdrive/assets/fonts/*.otf")

openpilot/selfdrive/ui/body/layouts/onroad.py

@@ -1,5 +1,5 @@
 import time
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 
 from openpilot.system.ui.lib.application import gui_app, FontWeight
 from openpilot.system.ui.widgets import Widget

openpilot/selfdrive/ui/mici/layouts/home.py

@@ -2,7 +2,7 @@
 import time
 
 from openpilot.cereal import log
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from collections.abc import Callable
 from openpilot.system.ui.widgets import Widget
 from openpilot.system.ui.widgets.layouts import HBoxLayout

openpilot/selfdrive/ui/mici/layouts/main.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import openpilot.cereal.messaging as messaging
 from openpilot.selfdrive.ui.mici.layouts.home import MiciHomeLayout
 from openpilot.selfdrive.ui.mici.layouts.settings.settings import SettingsLayout

openpilot/selfdrive/ui/mici/layouts/offroad_alerts.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import re
 import threading
 import time

openpilot/selfdrive/ui/mici/layouts/onboarding.py

@@ -1,7 +1,7 @@
 import math
 import numpy as np
 import qrcode
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from collections.abc import Callable
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.system.ui.lib.application import FontWeight, gui_app

openpilot/selfdrive/ui/mici/layouts/settings/device.py

@@ -1,5 +1,5 @@
 import os
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from collections.abc import Callable
 
 from openpilot.common.basedir import BASEDIR

openpilot/selfdrive/ui/mici/layouts/settings/firehose.py

@@ -1,7 +1,7 @@
 import requests
 import threading
 import time
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 
 from openpilot.common.api import api_get
 from openpilot.common.params import Params

openpilot/selfdrive/ui/mici/layouts/settings/network/__init__.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 
 from openpilot.selfdrive.ui.mici.layouts.settings.network.wifi_ui import WifiIcon
 from openpilot.selfdrive.ui.mici.widgets.button import BigButton

openpilot/selfdrive/ui/mici/layouts/settings/network/wifi_ui.py

@@ -1,6 +1,6 @@
 import math
 import numpy as np
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from collections.abc import Callable
 
 from openpilot.common.swaglog import cloudlog

openpilot/selfdrive/ui/mici/layouts/settings/software.py

@@ -1,6 +1,6 @@
 import subprocess
 import threading
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from enum import IntEnum
 from collections.abc import Callable
 

openpilot/selfdrive/ui/mici/onroad/__init__.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 
 SIDE_PANEL_WIDTH = 60
 

openpilot/selfdrive/ui/mici/onroad/alert_renderer.py

@@ -1,7 +1,7 @@
 import time
 from enum import StrEnum
 from typing import NamedTuple
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import random
 import string
 from dataclasses import dataclass

openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -1,5 +1,5 @@
 import numpy as np
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from openpilot.cereal import log
 from opendbc.car.structs import car
 from msgq.visionipc import VisionStreamType

openpilot/selfdrive/ui/mici/onroad/cameraview.py

@@ -1,6 +1,6 @@
 import platform
 import numpy as np
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 
 from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
@@ -120,12 +120,14 @@ def __init__(self, name: str, stream_type: VisionStreamType):
 
     self._texture_needs_update = True
     self.last_connection_attempt: float = 0.0
-    self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
-    self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
-    self._engaged_loc = rl.get_shader_location(self.shader, "engaged")
-    self._engaged_val = rl.ffi.new("int[1]", [1])
-    self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
-    self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0])
+    self.shader = rl.Shader()
+    if not rl.using_cpu_backend():
+      self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
+      self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
+      self._engaged_loc = rl.get_shader_location(self.shader, "engaged")
+      self._engaged_val = rl.ffi.new("int[1]", [1])
+      self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
+      self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0])
 
     self.frame: VisionBuf | None = None
     self.texture_y: rl.Texture | None = None
@@ -138,7 +140,7 @@ def __init__(self, name: str, stream_type: VisionStreamType):
     self._placeholder_color: rl.Color | None = None
 
     # Initialize EGL for zero-copy rendering on TICI
-    if TICI:
+    if TICI and not rl.using_cpu_backend():
       if not init_egl():
         raise RuntimeError("Failed to initialize EGL")
 
@@ -185,12 +187,12 @@ def close(self) -> None:
     self._clear_textures()
 
     # Clean up EGL texture
-    if TICI and self.egl_texture:
+    if TICI and not rl.using_cpu_backend() and self.egl_texture:
       rl.unload_texture(self.egl_texture)
       self.egl_texture = None
 
     # Clean up shader
-    if self.shader and self.shader.id:
+    if self.shader.id:
       rl.unload_shader(self.shader)
       self.shader.id = 0
 
@@ -260,7 +262,10 @@ def _render(self, rect: rl.Rectangle, /):
     dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
 
     # Render with appropriate method
-    if TICI:
+    if rl.using_cpu_backend():
+      rl.draw_nv12(self.frame, dst_rect, ui_state.status != UIStatus.DISENGAGED,
+                   self._stream_type == VisionStreamType.VISION_STREAM_DRIVER)
+    elif TICI:
       self._render_egl(src_rect, dst_rect)
     else:
       self._render_textures(src_rect, dst_rect)
@@ -384,6 +389,8 @@ def _complete_switch(self) -> None:
 
   def _initialize_textures(self):
     self._clear_textures()
+    if rl.using_cpu_backend():
+      return
     if not TICI:
       self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
         int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
@@ -400,7 +407,7 @@ def _clear_textures(self):
       self.texture_uv = None
 
     # Clean up EGL resources
-    if TICI:
+    if TICI and not rl.using_cpu_backend():
       for data in self.egl_images.values():
         destroy_egl_image(data)
       self.egl_images = {}

openpilot/selfdrive/ui/mici/onroad/confidence_ball.py

@@ -1,5 +1,5 @@
 import math
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from openpilot.selfdrive.ui.mici.onroad import SIDE_PANEL_WIDTH
 from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
 from openpilot.system.ui.widgets import Widget

openpilot/selfdrive/ui/mici/onroad/driver_camera_dialog.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from openpilot.cereal import log, messaging
 from msgq.visionipc import VisionStreamType
 from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView

openpilot/selfdrive/ui/mici/onroad/driver_state.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import numpy as np
 import math
 from openpilot.cereal import log

openpilot/selfdrive/ui/mici/onroad/hud_renderer.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from dataclasses import dataclass
 from openpilot.common.constants import CV
 from openpilot.selfdrive.ui.mici.onroad.torque_bar import TorqueBar

openpilot/selfdrive/ui/mici/onroad/model_renderer.py

@@ -1,6 +1,6 @@
 import colorsys
 import numpy as np
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from openpilot.cereal import messaging
 from opendbc.car.structs import car
 from dataclasses import dataclass, field

openpilot/selfdrive/ui/mici/onroad/torque_bar.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict
 
 import numpy as np
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from opendbc.car import ACCELERATION_DUE_TO_GRAVITY
 from openpilot.selfdrive.ui.mici.onroad import blend_colors
 from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus

openpilot/selfdrive/ui/mici/widgets/button.py

@@ -1,5 +1,5 @@
 import math
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from typing import TYPE_CHECKING, Union
 from enum import Enum
 from collections.abc import Callable

openpilot/selfdrive/ui/mici/widgets/dialog.py

@@ -1,6 +1,6 @@
 import abc
 import math
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 from typing import Union
 from collections.abc import Callable
 from openpilot.system.ui.widgets.nav_widget import NavWidget

openpilot/selfdrive/ui/mici/widgets/pairing_dialog.py

@@ -1,4 +1,4 @@
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import qrcode
 import numpy as np
 import time

openpilot/selfdrive/ui/tests/profile_onroad.py

@@ -2,7 +2,7 @@
 import os
 import time
 import cProfile
-import pyray as rl
+from openpilot.system.ui.lib import raylib as rl
 import numpy as np
 
 from msgq.visionipc import VisionIpcServer, VisionStreamType
@@ -76,7 +76,7 @@ def mock_update(timeout=None):
   if args.max_seconds:
     message_chunks = message_chunks[:int(args.max_seconds * FPS)]
 
-  print("Initializing UI with GPU rendering...")
+  print(f"Initializing UI with {'CPU' if rl.using_cpu_backend() else 'GPU'} rendering...")
 
   if args.headless:
     os.environ['SDL_VIDEODRIVER'] = 'dummy'