replace submodules with packages

.github/labeler.yaml

@@ -4,7 +4,7 @@ CI / testing:
 
 car:
   - changed-files:
-    - any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}'
+    - any-glob-to-all-files: 'openpilot/selfdrive/car/**'
 
 simulation:
   - changed-files:

.gitmodules

@@ -1,18 +1,3 @@
-[submodule "panda"]
-  path = panda
-  url = ../../commaai/panda.git
-[submodule "opendbc"]
-  path = opendbc_repo
-  url = ../../commaai/opendbc.git
-[submodule "msgq"]
-  path = msgq_repo
-  url = ../../commaai/msgq.git
-[submodule "rednose_repo"]
-  path = rednose_repo
-  url = ../../commaai/rednose.git
-[submodule "teleoprtc_repo"]
-  path = teleoprtc_repo
-  url = ../../commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
   url = https://github.com/tinygrad/tinygrad.git

SConstruct

@@ -7,6 +7,11 @@ import shlex
 import importlib
 import numpy as np
 
+import msgq as msgq_package
+import opendbc
+import panda
+import rednose as rednose_package
+
 import SCons.Errors
 from SCons.Defaults import _stripixes
 
@@ -27,20 +32,16 @@ AddOption('--minimal',
           default=(not COMMA_HARDWARE and not release),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-submodule_python_paths = [
+python_paths = [
   Dir("#").abspath,
-  Dir("#msgq_repo").abspath,
-  Dir("#opendbc_repo").abspath,
-  Dir("#rednose_repo").abspath,
-  Dir("#teleoprtc_repo").abspath,
   Dir("#tinygrad_repo").abspath,
 ]
-for p in reversed(submodule_python_paths):
+for p in reversed(python_paths):
   if p not in sys.path:
     sys.path.insert(0, p)
 
 if external_pythonpath := os.environ.get("PYTHONPATH"):
-  submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
+  python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in python_paths]
 
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
@@ -121,7 +122,7 @@ def _libflags(target, source, env, for_signature):
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": os.pathsep.join(submodule_python_paths),
+    "PYTHONPATH": os.pathsep.join(python_paths),
     "ACADOS_SOURCE_DIR": acados.DIR,
     "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
     "TERA_PATH": acados.TERA_PATH
@@ -144,28 +145,27 @@ env = Environment(
   CXXFLAGS=["-std=c++1z"],
   CPPPATH=[
     "#openpilot",
-    "#msgq_repo",            # #include "msgq/..."
-    "#opendbc_repo",         # #include "opendbc/..."
-    "#rednose_repo",         # #include "rednose/..."
-    "#rednose_repo/rednose", # #include "logger/..." (rednose package root)
+    msgq_package.INCLUDE_PATH,
+    opendbc.INCLUDE_PATH,
+    panda.INCLUDE_PATH,
+    rednose_package.INCLUDE_PATH,
+    os.path.dirname(rednose_package.__file__), # #include "logger/..."
     "#openpilot/cereal/gen/cpp",
     acados_include_dirs,
     [x.INCLUDE_DIR for x in pkgs],
     "#",
   ],
   LIBPATH=[
     "#openpilot/common",
-    "#msgq_repo",
     "#openpilot/selfdrive/pandad",
-    "#rednose_repo/rednose/helpers",
     [x.LIB_DIR for x in pkgs],
   ],
   RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  REDNOSE_ROOT="#rednose_repo",
+  REDNOSE_ROOT=rednose_package.INCLUDE_PATH,
   tools=["default", "cython", "compilation_db", "rednose_filter"],
-  toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+  toolpath=[rednose_package.SCONS_TOOL_PATH],
 )
 # SCons' Darwin linker tool doesn't define the variables used to expand RPATH.
 if arch == "Darwin":
@@ -256,24 +256,20 @@ Import('_common')
 common = [_common, 'json11', 'zmq']
 Export('common')
 
-# Build messaging (cereal + msgq + socketmaster + their dependencies)
-# Enable swaglog include in submodules
-env_swaglog = env.Clone()
-env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
-SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+# Build messaging (cereal + installed msgq + socketmaster)
+msgq = File(msgq_package.LIB_PATH)
+visionipc = File(msgq_package.VISIONIPC_LIB_PATH)
+msgq_python = File(msgq_package.PYTHON_LIB_PATH)
+Export('msgq', 'visionipc', 'msgq_python')
 
 SConscript(['openpilot/cereal/SConscript'])
 
-Import('socketmaster', 'msgq')
+Import('socketmaster')
 messaging = [socketmaster, msgq, 'capnp', 'kj',]
 Export('messaging')
 
-
-# Build other submodules
-SConscript(['panda/SConscript'])
-
-# Build rednose library
-SConscript(['rednose_repo/rednose/SConscript'])
+rednose = File(rednose_package.LIB_PATH)
+Export('rednose')
 
 # Build system services
 SConscript([

launch_chffrplus.sh

@@ -70,12 +70,7 @@ function launch {
   ln -sfn $(pwd) /data/pythonpath
   export PYTHONPATH="$PWD"
 
-  # submodule package symlinks for PYTHONPATH imports on device.
-  # on PC these come from editable installs via pyproject.toml / uv.
-  ln -sfn msgq_repo/msgq msgq
-  ln -sfn opendbc_repo/opendbc opendbc
-  ln -sfn rednose_repo/rednose rednose
-  ln -sfn teleoprtc_repo/teleoprtc teleoprtc
+  # tinygrad remains vendored on device.
   ln -sfn tinygrad_repo/tinygrad tinygrad
 
   # hardware specific init

openpilot/cereal/SConscript

@@ -1,15 +1,19 @@
+import os
+import opendbc
+
 Import('env', 'common', 'msgq')
 
 cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 
 # Build cereal
 schema_files = ['log.capnp', 'deprecated.capnp', 'custom.capnp']
-car_capnp = '#opendbc_repo/opendbc/car/car.capnp'
+car_schema_dir = os.path.join(os.path.dirname(opendbc.__file__), 'car')
+car_capnp = os.path.join(car_schema_dir, 'car.capnp')
 all_output = schema_files + ['car.capnp']
 env.Command([f'gen/cpp/{s}.c++' for s in all_output] + [f'gen/cpp/{s}.h' for s in all_output],
             schema_files + [car_capnp],
-            f"capnpc --src-prefix={cereal_dir.path} --src-prefix=opendbc_repo/opendbc/car --import-path=opendbc_repo/opendbc/car $SOURCES -o c++:{gen_dir.path}/cpp/")
+            f"capnpc --src-prefix={cereal_dir.path} --src-prefix={car_schema_dir} --import-path={car_schema_dir} $SOURCES -o c++:{gen_dir.path}/cpp/")
 
 cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in all_output])
 

openpilot/selfdrive/car/card.py

@@ -105,6 +105,7 @@ def __init__(self, CI=None, RI=None) -> None:
       # continue onto next fingerprinting step in pandad
       self.params.put_bool("FirmwareQueryDone", True, block=True)
     else:
+      assert RI is not None
       self.CI, self.CP = CI, CI.CP
       self.RI = RI
 

openpilot/selfdrive/pandad/SConscript

@@ -1,6 +1,9 @@
 Import('env', 'arch', 'common', 'messaging')
 
+import panda as panda_package
+
 if arch != "Darwin":
+  env.Append(CXXFLAGS=['-DPANDA_FW_PATH=\\"%s\\"' % panda_package.FW_PATH])
   libs = [common, messaging, 'pthread']
   panda = env.Library('panda', ['panda.cc', 'spi.cc'])
 

openpilot/selfdrive/pandad/panda.cc

@@ -116,7 +116,7 @@ std::optional<std::string> Panda::get_serial() {
 bool Panda::up_to_date() {
   if (auto fw_sig = get_firmware_version()) {
     for (auto fn : { "panda.bin.signed", "panda_h7.bin.signed" }) {
-      auto content = util::read_file(std::string("../../../panda/board/obj/") + fn);
+      auto content = util::read_file(std::string(PANDA_FW_PATH) + fn);
       if (content.size() >= fw_sig->size() &&
           memcmp(content.data() + content.size() - fw_sig->size(), fw_sig->data(), fw_sig->size()) == 0) {
         return true;

openpilot/selfdrive/pandad/panda.h

@@ -11,8 +11,8 @@
 
 #include "openpilot/cereal/gen/cpp/car.capnp.h"
 #include "openpilot/cereal/gen/cpp/log.capnp.h"
-#include "panda/board/health.h"
-#include "panda/board/can.h"
+#include "panda/health.h"
+#include "panda/can.h"
 #include "selfdrive/pandad/panda_comms.h"
 
 #define USB_TX_SOFT_LIMIT   (0x100U)

openpilot/selfdrive/pandad/spi.cc

@@ -11,7 +11,7 @@
 #include "common/util.h"
 #include "common/timing.h"
 #include "common/swaglog.h"
-#include "panda/board/comms_definitions.h"
+#include "panda/comms_definitions.h"
 #include "selfdrive/pandad/panda_comms.h"
 
 

openpilot/selfdrive/test/process_replay/migration.py

@@ -8,7 +8,7 @@
 from openpilot.cereal import messaging, log
 from opendbc.car.structs import car
 from opendbc.car.fingerprints import MIGRATION
-from opendbc.car.toyota.values import EPS_SCALE, ToyotaSafetyFlags
+from opendbc.car.toyota.values import CAR as TOYOTA, EPS_SCALE, ToyotaSafetyFlags
 from opendbc.car.ford.values import CAR as FORD, FordFlags, FordSafetyFlags
 from opendbc.car.hyundai.values import HyundaiSafetyFlags
 from opendbc.car.gm.values import GMSafetyFlags
@@ -306,8 +306,8 @@ def migrate_carOutput(msgs):
 def migrate_pandaStates(msgs):
   # TODO: safety param migration should be handled automatically
   safety_param_migration = {
-    "TOYOTA_PRIUS": EPS_SCALE["TOYOTA_PRIUS"] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
-    "TOYOTA_RAV4": EPS_SCALE["TOYOTA_RAV4"] | ToyotaSafetyFlags.ALT_BRAKE,
+    "TOYOTA_PRIUS": EPS_SCALE[TOYOTA.TOYOTA_PRIUS] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
+    "TOYOTA_RAV4": EPS_SCALE[TOYOTA.TOYOTA_RAV4] | ToyotaSafetyFlags.ALT_BRAKE,
     "KIA_EV6": HyundaiSafetyFlags.EV_GAS | HyundaiSafetyFlags.CANFD_LKA_STEER_MSG,
     "CHEVROLET_VOLT": GMSafetyFlags.EV,
     "CHEVROLET_BOLT_EUV": GMSafetyFlags.EV | GMSafetyFlags.HW_CAM,

openpilot/selfdrive/test/scons_build_test.sh

@@ -11,7 +11,6 @@ cd $BASEDIR
 # helpful commands:
 # scons -Q --tree=derived
 
-cd $BASEDIR/opendbc_repo/
 scons --clean
 scons --no-cache --random
 if ! scons -q; then

openpilot/selfdrive/test/setup_device_ci.sh

@@ -145,12 +145,8 @@ else
   safe_checkout
 fi
 
-# submodule package symlinks for PYTHONPATH imports on device (same as launch_chffrplus.sh)
+# tinygrad remains vendored on device (same as launch_chffrplus.sh)
 cd $TEST_DIR
-ln -sfn msgq_repo/msgq msgq
-ln -sfn opendbc_repo/opendbc opendbc
-ln -sfn rednose_repo/rednose rednose
-ln -sfn teleoprtc_repo/teleoprtc teleoprtc
 ln -sfn tinygrad_repo/tinygrad tinygrad
 
 echo "$TEST_DIR synced with $GIT_COMMIT, t=$SECONDS"

openpilot/system/webrtc/webrtcd.py

@@ -272,7 +272,7 @@ async def stop(self):
   async def get_answer(self):
     return await self.stream.start()
 
-  def message_handler(self, message: bytes):
+  def message_handler(self, message: bytes | str):
     try:
       payload = json.loads(message) if isinstance(message, (bytes, str)) else None
       if isinstance(payload, dict):
@@ -309,7 +309,7 @@ def message_handler(self, message: bytes):
             if msg_type not in self.incoming_bridge_services:
               return
             if self.incoming_bridge is not None:
-              self.incoming_bridge.send(message)
+              self.incoming_bridge.send(message.encode() if isinstance(message, str) else message)
     except Exception:
       self.logger.exception("Cereal incoming proxy failure")
 

openpilot/tools/cabana/SConscript

@@ -4,6 +4,7 @@ import shutil
 
 import bootstrap_icons
 import libusb
+import opendbc
 
 Import('env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'replay_lib', 'ffmpeg_libs')
 
@@ -74,7 +75,7 @@ cabana_env['CPPPATH'] += [libusb.INCLUDE_DIR]
 cabana_env['LIBPATH'] += [libusb.LIB_DIR]
 
 cabana_libs = [cereal, messaging, visionipc, replay_lib] + ffmpeg_libs + ['usb-1.0'] + base_libs
-opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % (cabana_env.Dir("../../../opendbc_repo/opendbc/dbc").abspath)
+opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % opendbc.DBC_PATH
 cabana_env['CXXFLAGS'] += [opendbc_path]
 
 # embed the bootstrap icons SVG into the binary
@@ -126,4 +127,4 @@ output_json_file = 'openpilot/tools/cabana/dbc/car_fingerprint_to_dbc.json'
 generate_dbc = cabana_env.Command('#' + output_json_file,
                                   ['dbc/generate_dbc_json.py'],
                                   "python3 openpilot/tools/cabana/dbc/generate_dbc_json.py --out " + output_json_file)
-cabana_env.Depends(generate_dbc, ["#openpilot/common", '#opendbc_repo', "#openpilot/cereal", "#msgq_repo"])
+cabana_env.Depends(generate_dbc, ["#openpilot/common", "#openpilot/cereal"])

openpilot/tools/cabana/panda.h

@@ -15,8 +15,8 @@
 
 #include "openpilot/cereal/gen/cpp/car.capnp.h"
 #include "openpilot/cereal/gen/cpp/log.capnp.h"
-#include "panda/board/health.h"
-#include "panda/board/can.h"
+#include "panda/health.h"
+#include "panda/can.h"
 
 #define USB_TX_SOFT_LIMIT   (0x100U)
 #define USBPACKET_MAX_SIZE  (0x40)

openpilot/tools/jotpluggler/SConscript

@@ -3,6 +3,7 @@ import subprocess
 import bootstrap_icons
 import imgui
 import libusb
+import opendbc
 from opendbc import get_generated_dbcs
 from opendbc.car import Bus
 from opendbc.car.fingerprints import MIGRATION
@@ -17,6 +18,7 @@ jot_env["CPPPATH"] += [imgui.INCLUDE_DIR, libusb.INCLUDE_DIR]
 jot_env["CXXFLAGS"] += [
   "-DGLFW_INCLUDE_NONE",
   '-DJOTP_REPO_ROOT=\'"%s"\'' % os.path.realpath(BASEDIR),
+  '-DOPENDBC_DBC_PATH=\'"%s"\'' % opendbc.DBC_PATH,
   '-DBOOTSTRAP_ICONS_TTF=\'"%s"\'' % bootstrap_icons.TTF_PATH,
 ]
 
@@ -85,6 +87,7 @@ def generate_event_extractors(target, source, env):
     "python3",
     "openpilot/tools/jotpluggler/generate_event_extractors.py",
     os.path.realpath(BASEDIR),
+    os.path.dirname(opendbc.__file__),
     str(target[0]),
   ])
   return None
@@ -95,8 +98,8 @@ event_extractors = jot_env.Command("generated_event_extractors.h", [
     "generate_event_extractors.py",
     jot_env.Glob("#openpilot/cereal/*.capnp"),
     jot_env.Glob("#openpilot/cereal/include/*.capnp"),
-    "#opendbc_repo/opendbc/car/car.capnp",
-    "#opendbc_repo/opendbc/car/include/c++.capnp",
+    os.path.join(os.path.dirname(opendbc.__file__), "car", "car.capnp"),
+    os.path.join(os.path.dirname(opendbc.__file__), "car", "include", "c++.capnp"),
   ],
   generate_event_extractors,
 )

openpilot/tools/jotpluggler/generate_event_extractors.py

@@ -391,14 +391,15 @@ def generate(self):
 
 
 if __name__ == "__main__":
-  if len(sys.argv) != 3:
-    print(f"usage: {sys.argv[0]} <repo-root> <output>", file=sys.stderr)
+  if len(sys.argv) != 4:
+    print(f"usage: {sys.argv[0]} <repo-root> <opendbc-root> <output>", file=sys.stderr)
     sys.exit(2)
 
   repo_root = Path(sys.argv[1]).resolve()
-  output = Path(sys.argv[2])
+  opendbc_root = Path(sys.argv[2]).resolve()
+  output = Path(sys.argv[3])
   capnp.remove_import_hook()
-  log = capnp.load(str(repo_root / "openpilot" / "cereal" / "log.capnp"), imports=[str(repo_root / "opendbc_repo" / "opendbc" / "car")])
+  log = capnp.load(str(repo_root / "openpilot" / "cereal" / "log.capnp"), imports=[str(opendbc_root / "car")])
   generated = Generator(log.Event.schema).generate()
   output.parent.mkdir(parents=True, exist_ok=True)
   output.write_text(generated)

openpilot/tools/jotpluggler/layout.cc

@@ -276,7 +276,7 @@ std::string default_dbc_template() {
 DbcEditorSource resolve_dbc_editor_source(const std::string &dbc_name) {
   const fs::path generated_dbc_dir = repo_root() / "openpilot" / "tools" / "jotpluggler" / "generated_dbcs";
   const std::array<DbcEditorSource, 2> candidates = {{
-    {.path = repo_root() / "opendbc_repo" / "opendbc" / "dbc" / (dbc_name + ".dbc"), .kind = DbcEditorState::SourceKind::Opendbc},
+    {.path = fs::path(OPENDBC_DBC_PATH) / (dbc_name + ".dbc"), .kind = DbcEditorState::SourceKind::Opendbc},
     {.path = generated_dbc_dir / (dbc_name + ".dbc"), .kind = DbcEditorState::SourceKind::Generated},
   }};
   for (const DbcEditorSource &candidate : candidates) {

openpilot/tools/jotpluggler/runtime.cc

@@ -8,7 +8,7 @@
 #include "imgui_impl_opengl3_loader.h"
 #include "implot.h"
 #include "common/yuv.h"
-#include "msgq_repo/msgq/ipc.h"
+#include "msgq/ipc.h"
 #include "tools/replay/framereader.h"
 
 #include <GLFW/glfw3.h>