rewrite bootlog in Python

openpilot/common/hardware/base.py

@@ -81,6 +81,9 @@ def uninstall(self):
   def get_os_version(self):
     return None
 
+  def get_init_logs(self) -> dict[str, bytes]:
+    return {}
+
   @abstractmethod
   def get_device_type(self):
     pass

openpilot/common/hardware/comma/hardware.py

@@ -89,6 +89,35 @@ def get_os_version(self):
   def get_device_type(self):
     return get_device_type()
 
+  def get_init_logs(self) -> dict[str, bytes]:
+    def read_file(path: str) -> bytes:
+      try:
+        return Path(path).read_bytes()
+      except OSError:
+        return b""
+
+    def check_output(command: list[str]) -> bytes:
+      try:
+        return subprocess.check_output(command)
+      except (OSError, subprocess.CalledProcessError):
+        return b""
+
+    logs = {
+      "/BUILD": read_file("/BUILD"),
+      "lsblk": check_output(["lsblk", "-o", "NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL"]),
+      "SOM ID": read_file("/sys/devices/platform/vendor/vendor:gpio-som-id/som_id"),
+    }
+
+    logs["boot slot"] = check_output(["abctl", "--boot_slot"]).split(b"\n", 1)[0]
+    logs["boot temp"] = read_file("/dev/disk/by-partlabel/ssd").rstrip(b"\0\r\n")
+
+    for part in ("xbl", "abl", "aop", "devcfg", "xbl_config"):
+      for slot in ("a", "b"):
+        partition = f"{part}_{slot}"
+        logs[partition] = check_output(["sha256sum", f"/dev/disk/by-partlabel/{partition}"]).split(b" ", 1)[0]
+
+    return logs
+
   def reboot(self, reason=None):
     subprocess.check_output(["sudo", "reboot"])
 

openpilot/common/params.py

@@ -1,6 +1,7 @@
 import sys
 import json
 import ctypes
+import functools
 import weakref
 import builtins
 import datetime
@@ -15,6 +16,7 @@ class ParamKeyFlag(IntFlag):
   CLEAR_ON_MANAGER_START = 0x04
   CLEAR_ON_ONROAD_TRANSITION = 0x08
   CLEAR_ON_OFFROAD_TRANSITION = 0x10
+  DONT_LOG = 0x20
   DEVELOPMENT_ONLY = 0x40
   CLEAR_ON_IGNITION_ON = 0x80
   ALL = 0xFFFFFFFF
@@ -77,6 +79,12 @@ def checked(*call_args):
 params_keys_size = _bind("params_keys_size", [ParamsHandle], ctypes.c_size_t)
 params_key_at = _bind("params_key_at", [ParamsHandle, ctypes.c_size_t], ParamsBuffer)
 
+
+@functools.cache
+def _params_get_key_flag():
+  return _bind("params_get_key_flag", [ParamsHandle, ctypes.c_char_p], ctypes.c_uint)
+
+
 PYTHON_2_CPP = {
   (str, ParamKeyType.STRING): lambda v: v,
   (builtins.bool, ParamKeyType.BOOL): lambda v: "1" if v else "0",
@@ -186,6 +194,9 @@ def get_param_path(self, key=""):
   def get_type(self, key):
     return ParamKeyType(params_get_key_type(self.p, self.check_key(key)))
 
+  def get_flag(self, key):
+    return ParamKeyFlag(_params_get_key_flag()(self.p, self.check_key(key)))
+
   def all_keys(self):
     keys = []
     for i in range(params_keys_size(self.p)):

openpilot/common/params_c.cc

@@ -91,6 +91,12 @@ int params_get_key_type(ParamsHandle *handle, const char *key) noexcept {
   });
 }
 
+unsigned int params_get_key_flag(ParamsHandle *handle, const char *key) noexcept {
+  return translate_exceptions(0U, [&]() {
+    return static_cast<unsigned int>(handle->params.getKeyFlag(key));
+  });
+}
+
 ParamsBuffer params_get_default(ParamsHandle *handle, const char *key) noexcept {
   return translate_exceptions(ParamsBuffer{nullptr, 0}, [&]() {
     auto value = handle->params.getKeyDefaultValue(key);

openpilot/common/tests/test_params.py

@@ -144,3 +144,7 @@ def test_params_get_type(self):
     now = datetime.datetime.now(datetime.UTC)
     self.params.put("InstallDate", now, block=True)
     assert self.params.get("InstallDate") == now
+
+  def test_params_get_flag(self):
+    assert self.params.get_flag("AccessToken") & ParamKeyFlag.DONT_LOG
+    assert not self.params.get_flag("DongleId") & ParamKeyFlag.DONT_LOG

openpilot/system/loggerd/SConscript

@@ -16,4 +16,3 @@ libs.insert(0, logger_lib)
 
 env.Program('loggerd', ['loggerd.cc'], LIBS=libs, FRAMEWORKS=frameworks)
 env.Program('encoderd', ['encoderd.cc'], LIBS=libs, FRAMEWORKS=frameworks)
-env.Program('bootlog.cc', LIBS=libs, FRAMEWORKS=frameworks)

openpilot/system/loggerd/bootlog.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import subprocess
+import time
+from pathlib import Path
+
+import openpilot.cereal.messaging as messaging
+import zstandard as zstd
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.params import Params
+from openpilot.common.utils import LOG_COMPRESSION_LEVEL
+from openpilot.system.loggerd.logger import build_init_data, get_identifier
+
+
+def _read_files(path: str | Path) -> dict[str, bytes]:
+  try:
+    entries = list(Path(path).iterdir())
+  except OSError:
+    return {}
+  files = {}
+  for entry in entries:
+    if entry.is_dir():
+      continue
+    try:
+      files[entry.name] = entry.read_bytes()
+    except OSError:
+      pass
+  return files
+
+
+def build_boot_log() -> bytes:
+  msg = messaging.new_message("boot", valid=True)
+  boot = msg.boot
+  boot.wallTimeNanos = time.time_ns()
+
+  pstore = _read_files("/sys/fs/pstore")
+  pstore_entries = boot.pstore.init("entries", len(pstore))
+  for entry, (key, value) in zip(pstore_entries, sorted(pstore.items()), strict=True):
+    entry.key = key
+    entry.value = value
+
+  command = '[ -x "$(command -v journalctl)" ] && journalctl -b -n 2000 -o short-monotonic --no-pager'
+  command_entry = boot.commands.init("entries", 1)[0]
+  command_entry.key = command
+  try:
+    command_entry.value = subprocess.check_output(command, shell=True)
+  except (OSError, subprocess.CalledProcessError):
+    command_entry.value = b""
+
+  try:
+    boot.launchLog = Path("/tmp/launch_log").read_bytes().decode("utf-8", "replace")
+  except OSError:
+    boot.launchLog = ""
+  return msg.to_bytes()
+
+
+def create_bootlog(params_path: str = "") -> Path:
+  identifier = get_identifier("BootCount")
+  boot_dir = Path(Paths.log_root()) / "boot"
+  path = boot_dir / f"{identifier}.zst"
+  print(f"bootlog to {path}")
+
+  boot_dir.mkdir(mode=0o775, parents=True, exist_ok=True)
+  with zstd.open(path, "wb", cctx=zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)) as writer:
+    writer.write(build_init_data(params_path))
+    writer.write(build_boot_log())
+
+  Params().put("CurrentBootlog", identifier, block=True)
+  return path
+
+
+if __name__ == "__main__":
+  create_bootlog()

openpilot/system/loggerd/logger.py

@@ -0,0 +1,89 @@
+import os
+import secrets
+import subprocess
+import time
+from pathlib import Path
+
+import openpilot.cereal.messaging as messaging
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.params import ParamKeyFlag, Params
+from openpilot.common.version import get_version
+
+
+def _read_text(path: str | Path) -> str:
+  try:
+    return Path(path).read_text(errors="replace")
+  except OSError:
+    return ""
+
+
+def _raw_params(params: Params) -> dict[str, bytes]:
+  values = {}
+  try:
+    entries = list(Path(params.get_param_path()).iterdir())
+  except OSError:
+    return values
+
+  for entry in entries:
+    if entry.is_dir():
+      continue
+    try:
+      value = entry.read_bytes()
+    except OSError:
+      continue
+    values[entry.name] = value
+  return values
+
+
+def build_init_data(params_path: str = "") -> bytes:
+  msg = messaging.new_message("initData", valid=True)
+  init = msg.initData
+
+  init.wallTimeNanos = time.time_ns()
+  init.version = get_version()
+  init.dirty = os.getenv("CLEAN") is None
+  init.deviceType = HARDWARE.get_device_type()
+
+  init.kernelArgs = _read_text("/proc/cmdline").split()
+  init.kernelVersion = _read_text("/proc/version")
+  init.osVersion = _read_text("/VERSION")
+
+  params = Params(params_path)
+  params_map = _raw_params(params)
+  init.gitCommit = params_map.get("GitCommit", b"").decode("utf-8", "replace")
+  init.gitCommitDate = params_map.get("GitCommitDate", b"").decode("utf-8", "replace")
+  init.gitBranch = params_map.get("GitBranch", b"").decode("utf-8", "replace")
+  init.gitRemote = params_map.get("GitRemote", b"").decode("utf-8", "replace")
+  init.passive = False
+  init.dongleId = params_map.get("DongleId", b"").decode("utf-8", "replace")
+
+  init.gitSrcCommit = _read_text(Path(BASEDIR) / "openpilot" / "git_src_commit")
+  init.gitSrcCommitDate = _read_text(Path(BASEDIR) / "openpilot" / "git_src_commit_date")
+
+  param_entries = init.params.init("entries", len(params_map))
+  for entry, (key, value) in zip(param_entries, sorted(params_map.items()), strict=True):
+    entry.key = key
+    entry.value = b"" if params.get_flag(key) & ParamKeyFlag.DONT_LOG else value
+
+  try:
+    df = subprocess.check_output(["df", "-h"])
+  except (OSError, subprocess.CalledProcessError):
+    df = b""
+  commands = {"df -h": df, **dict(sorted(HARDWARE.get_init_logs().items()))}
+  command_entries = init.commands.init("entries", len(commands))
+  for entry, (key, value) in zip(command_entries, commands.items(), strict=True):
+    entry.key = key
+    entry.value = value
+
+  return msg.to_bytes()
+
+
+def get_identifier(key: str) -> str:
+  params = Params()
+  try:
+    count = int(params.get(key) or 0)
+  except (TypeError, ValueError):
+    count = 0
+  params.put(key, count + 1, block=True)
+  return f"{count:08x}--{secrets.token_hex(5)}"