[experiment] browser clips

package.json

@@ -26,6 +26,8 @@
   },
   "dependencies": {
     "@commaai/my-comma-auth": "^1.4.1",
+    "@ffmpeg/core": "0.12.10",
+    "@ffmpeg/ffmpeg": "0.12.15",
     "@mapbox/mapbox-sdk": "^0.15.3",
     "@material-ui/core": "^1.5.1",
     "@material-ui/icons": "^1.1.1",

src/api/browserClips.js

@@ -0,0 +1,117 @@
+import { FFmpeg } from '@ffmpeg/ffmpeg';
+import coreURL from '@ffmpeg/core?url';
+import wasmURL from '@ffmpeg/core/wasm?url';
+import localforage from 'localforage';
+
+let ffmpeg;
+let loadPromise;
+const browserClipStorage = localforage.createInstance({ name: 'connect', storeName: 'browser_clip_cache' });
+
+function clipIndexKey(dongleId) {
+  return `clips:${dongleId}`;
+}
+
+function clipBlobKey(dongleId, clipId) {
+  return `blob:${dongleId}/${clipId}`;
+}
+
+export async function getBrowserClips(dongleId, route) {
+  const clips = await browserClipStorage.getItem(clipIndexKey(dongleId)).catch(() => []);
+  return Array.isArray(clips) ? clips.filter(clip => !route || clip.route === route) : [];
+}
+
+export async function saveBrowserClip(dongleId, clip, blob) {
+  const clipId = clip.cache_id || `${clip.requested_at}-${crypto.randomUUID()}`;
+  const storedClip = { ...clip, cache_id: clipId, local: true };
+  await browserClipStorage.setItem(clipBlobKey(dongleId, clipId), blob);
+  try {
+    const clips = await browserClipStorage.getItem(clipIndexKey(dongleId)).catch(() => []);
+    const withoutExisting = Array.isArray(clips) ? clips.filter(item => item.cache_id !== clipId) : [];
+    await browserClipStorage.setItem(clipIndexKey(dongleId), [storedClip, ...withoutExisting]);
+  } catch (error) {
+    await browserClipStorage.removeItem(clipBlobKey(dongleId, clipId)).catch(() => {});
+    throw error;
+  }
+  return storedClip;
+}
+
+export async function getBrowserClipBlob(dongleId, clipId) {
+  const blob = await browserClipStorage.getItem(clipBlobKey(dongleId, clipId));
+  if (!(blob instanceof Blob)) throw new Error('This browser clip is no longer available');
+  return blob;
+}
+
+export async function deleteBrowserClip(dongleId, clipId) {
+  await browserClipStorage.removeItem(clipBlobKey(dongleId, clipId));
+  const clips = await browserClipStorage.getItem(clipIndexKey(dongleId)).catch(() => []);
+  if (Array.isArray(clips)) {
+    await browserClipStorage.setItem(clipIndexKey(dongleId), clips.filter(clip => clip.cache_id !== clipId));
+  }
+}
+
+async function getFFmpeg() {
+  if (!ffmpeg) ffmpeg = new FFmpeg();
+  if (!loadPromise) loadPromise = ffmpeg.load({ coreURL, wasmURL });
+  await loadPromise;
+  return ffmpeg;
+}
+
+function selectedQcameraSegments(route, files, startTime, endTime) {
+  if (!route?.segment_numbers || !files) return [];
+  const routeStart = route.start_time_utc_millis;
+  return route.segment_numbers.flatMap((segmentNumber, index) => {
+    const segmentStart = (route.segment_start_times[index] - routeStart) / 1000;
+    const segmentEnd = (route.segment_end_times[index] - routeStart) / 1000;
+    if (segmentStart >= endTime || segmentEnd <= startTime) return [];
+    const file = files[`${route.fullname}--${segmentNumber}/qcameras`];
+    return file?.url ? [{ segmentStart, url: file.url }] : [];
+  });
+}
+
+export function browserClipAvailability(route, files, startTime, endTime) {
+  if (!route?.segment_numbers || !files || endTime <= startTime) return false;
+  const expected = route.segment_numbers.filter((segmentNumber, index) => {
+    const segmentStart = (route.segment_start_times[index] - route.start_time_utc_millis) / 1000;
+    const segmentEnd = (route.segment_end_times[index] - route.start_time_utc_millis) / 1000;
+    return segmentStart < endTime && segmentEnd > startTime;
+  });
+  return expected.length > 0 && expected.every(segmentNumber => files[`${route.fullname}--${segmentNumber}/qcameras`]?.url);
+}
+
+export async function createBrowserClip({ route, files, startTime, endTime, bitrate, speedup, onProgress }) {
+  const segments = selectedQcameraSegments(route, files, startTime, endTime);
+  if (!segments.length) throw new Error('Qcamera footage is not uploaded for this range');
+
+  const worker = await getFFmpeg();
+  const requestId = crypto.randomUUID();
+  const inputNames = segments.map((segment, index) => `qcamera-${requestId}-${index}.ts`);
+  const inputListName = `qcamera-${requestId}.txt`;
+  const outputName = `clip-${requestId}.mp4`;
+  const progressHandler = ({ progress }) => onProgress?.(Math.max(0, Math.min(1, progress)));
+  worker.on('progress', progressHandler);
+
+  try {
+    await Promise.all(segments.map(async ({ url }, index) => {
+      const response = await fetch(url);
+      if (!response.ok) throw new Error(`Could not download qcamera footage (${response.status})`);
+      await worker.writeFile(inputNames[index], new Uint8Array(await response.arrayBuffer()));
+    }));
+    await worker.writeFile(inputListName, new TextEncoder().encode(inputNames.map(name => `file '${name}'`).join('\n')));
+
+    const sourceOffset = Math.max(0, startTime - segments[0].segmentStart);
+    const sourceDuration = endTime - startTime;
+    const exitCode = await worker.exec([
+      '-ss', String(sourceOffset), '-f', 'concat', '-safe', '0', '-i', inputListName,
+      '-an', '-vf', `trim=duration=${sourceDuration},setpts=(PTS-STARTPTS)/${speedup}`, '-c:v', 'libx264', '-preset', 'superfast',
+      '-b:v', `${bitrate}M`, '-movflags', '+faststart', outputName,
+    ]);
+    if (exitCode !== 0) throw new Error('Browser transcoding failed');
+    const data = await worker.readFile(outputName);
+    return new Blob([data.buffer], { type: 'video/mp4' });
+  } finally {
+    worker.off('progress', progressHandler);
+    await Promise.all(inputNames.map(name => worker.deleteFile(name).catch(() => {})));
+    await worker.deleteFile(inputListName).catch(() => {});
+    await worker.deleteFile(outputName).catch(() => {});
+  }
+}

src/api/browserClips.test.js

@@ -0,0 +1,110 @@
+/* eslint-env jest */
+
+import { TextEncoder } from 'util';
+
+const ffmpegMock = {
+  deleteFile: jest.fn(() => Promise.resolve()),
+  exec: jest.fn(() => Promise.resolve(0)),
+  load: jest.fn(() => Promise.resolve()),
+  off: jest.fn(),
+  on: jest.fn(),
+  readFile: jest.fn(() => Promise.resolve(new Uint8Array([7, 8, 9]))),
+  writeFile: jest.fn(() => Promise.resolve()),
+};
+jest.mock('@ffmpeg/ffmpeg', () => ({ FFmpeg: jest.fn(() => ffmpegMock) }));
+jest.mock('@ffmpeg/core?url', () => 'ffmpeg-core.js', { virtual: true });
+jest.mock('@ffmpeg/core/wasm?url', () => 'ffmpeg-core.wasm', { virtual: true });
+jest.mock('localforage', () => {
+  const items = new Map();
+  const storage = {
+    getItem: jest.fn(key => Promise.resolve(items.get(key))),
+    removeItem: jest.fn(key => Promise.resolve(items.delete(key))),
+    setItem: jest.fn((key, value) => {
+      items.set(key, value);
+      return Promise.resolve(value);
+    }),
+  };
+  return { _items: items, createInstance: jest.fn(() => storage) };
+});
+
+import localforage from 'localforage';
+import {
+  browserClipAvailability, createBrowserClip, deleteBrowserClip, getBrowserClipBlob, getBrowserClips, saveBrowserClip,
+} from './browserClips';
+
+global.TextEncoder = TextEncoder;
+
+const route = {
+  fullname: 'dongle|2026-01-01--00-00-00',
+  start_time_utc_millis: 100000,
+  segment_numbers: [0, 1, 3],
+  segment_start_times: [100000, 160000, 280000],
+  segment_end_times: [160000, 220000, 340000],
+};
+
+function filesFor(...segments) {
+  return Object.fromEntries(segments.map(segment => [`${route.fullname}--${segment}/qcameras`, { url: `https://qcam/${segment}` }]));
+}
+
+describe('browser qcamera clips', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    localforage._items.clear();
+    global.crypto = { randomUUID: jest.fn(() => 'id') };
+  });
+
+  it('requires every qcamera segment intersecting the selected range', () => {
+    expect(browserClipAvailability(route, filesFor(0, 1), 10, 120)).toBe(true);
+    expect(browserClipAvailability(route, filesFor(0), 10, 120)).toBe(false);
+    expect(browserClipAvailability(route, filesFor(1), 60, 120)).toBe(true);
+    expect(browserClipAvailability(route, filesFor(0), 60, 120)).toBe(false);
+    expect(browserClipAvailability(route, filesFor(0, 1, 3), 120, 180)).toBe(false);
+  });
+
+  it('downloads in parallel, concatenates in route order, and preserves clip parameters', async () => {
+    const downloaded = {
+      'https://qcam/0': new Uint8Array([1, 2]),
+      'https://qcam/1': new Uint8Array([3, 4]),
+    };
+    global.fetch = jest.fn(url => Promise.resolve({
+      ok: true,
+      arrayBuffer: () => Promise.resolve(downloaded[url].buffer),
+    }));
+
+    const blob = await createBrowserClip({
+      route, files: filesFor(0, 1), startTime: 30, endTime: 90, bitrate: 8, speedup: 4,
+    });
+
+    expect(global.fetch).toHaveBeenCalledTimes(2);
+    expect(ffmpegMock.writeFile.mock.calls.slice(0, 2).map(call => [...call[1]])).toEqual(expect.arrayContaining([[1, 2], [3, 4]]));
+    expect(ffmpegMock.exec).toHaveBeenCalledWith(expect.arrayContaining([
+      '-ss', '30', '-f', 'concat', '-safe', '0', '-vf', 'trim=duration=60,setpts=(PTS-STARTPTS)/4', '-b:v', '8M', '-preset', 'superfast',
+    ]));
+    expect(blob.type).toBe('video/mp4');
+    expect(ffmpegMock.deleteFile).toHaveBeenCalledTimes(4);
+  });
+
+  it('cleans up temporary files when transcoding fails', async () => {
+    global.fetch = jest.fn(() => Promise.resolve({ ok: true, arrayBuffer: () => Promise.resolve(new Uint8Array([1]).buffer) }));
+    ffmpegMock.exec.mockResolvedValueOnce(1);
+
+    await expect(createBrowserClip({
+      route, files: filesFor(0), startTime: 0, endTime: 30, bitrate: 5, speedup: 1,
+    })).rejects.toThrow('Browser transcoding failed');
+    expect(ffmpegMock.off).toHaveBeenCalled();
+    expect(ffmpegMock.deleteFile).toHaveBeenCalledTimes(3);
+  });
+
+  it('persists, lists, reads, and deletes browser clips by device and route', async () => {
+    const blob = new Blob(['video'], { type: 'video/mp4' });
+    const first = await saveBrowserClip('dongle', { filename: 'first.mp4', requested_at: 1, route: 'route-a' }, blob);
+    await saveBrowserClip('dongle', { filename: 'second.mp4', requested_at: 2, route: 'route-b' }, blob);
+
+    expect(await getBrowserClips('dongle', 'route-a')).toEqual([first]);
+    expect(await getBrowserClipBlob('dongle', first.cache_id)).toBe(blob);
+
+    await deleteBrowserClip('dongle', first.cache_id);
+    expect(await getBrowserClips('dongle', 'route-a')).toEqual([]);
+    await expect(getBrowserClipBlob('dongle', first.cache_id)).rejects.toThrow('no longer available');
+  });
+});