diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index 73283ba8c331..e4b3afff0844 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -10,7 +10,11 @@ from PIL import Image from vllm.assets.base import get_vllm_public_assets -from vllm.assets.video import video_to_ndarrays, video_to_pil_images_list +from vllm.assets.video import ( + video_get_metadata, + video_to_ndarrays, + video_to_pil_images_list, +) from vllm.multimodal.media import ImageMediaIO, VideoMediaIO from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader @@ -112,6 +116,20 @@ def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext: assert np.nanmean(sim) > 0.99 +def test_opencv_video_metadata_matches_sampled_frame_timeline(tmp_path): + image_path = f"{tmp_path}/test_metadata_image.png" + Image.new("RGB", (8, 8), color=(255, 0, 0)).save(image_path) + video_path = f"{tmp_path}/test_metadata_video.mp4" + create_video_from_image(image_path, video_path, num_frames=10, fps=5.0) + + metadata = video_get_metadata(video_path, num_frames=4) + + assert metadata["fps"] == pytest.approx(5.0) + assert metadata["duration"] == pytest.approx(2.0) + assert metadata["frames_indices"] == [0, 3, 6, 9] + assert metadata["total_num_frames"] == 4 + + NUM_FRAMES = 10 FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3) FAKE_OUTPUT_2 = np.random.rand(NUM_FRAMES, 1280, 720, 3) diff --git a/vllm/assets/video.py b/vllm/assets/video.py index 72cd196c68f4..e06d033e0be1 100644 --- a/vllm/assets/video.py +++ b/vllm/assets/video.py @@ -15,6 +15,10 @@ from .base import get_cache_dir +def _sample_frame_indices(total_frames: int, num_frames: int) -> npt.NDArray: + return np.linspace(0, total_frames - 1, num_frames, dtype=int) + + @lru_cache def download_video_asset(filename: str) -> str: """ @@ -47,7 +51,7 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray: frames = [] num_frames = num_frames if num_frames > 0 else total_frames - frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) + frame_indices = _sample_frame_indices(total_frames, num_frames) for idx in range(total_frames): ok = cap.grab() # next img if not ok: @@ -86,13 +90,14 @@ def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]: if num_frames == -1 or num_frames > total_frames: num_frames = total_frames + frame_indices = _sample_frame_indices(total_frames, num_frames) metadata = { "total_num_frames": num_frames, - "fps": duration / num_frames, + "fps": fps, "duration": duration, "video_backend": "opencv", - "frames_indices": list(range(num_frames)), + "frames_indices": frame_indices.tolist(), # extra field used to control hf processor's video # sampling behavior "do_sample_frames": num_frames == total_frames,