diff --git a/src/app/main.cpp b/src/app/main.cpp index f8dc64c2ff..b824fb0c0d 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -273,6 +273,9 @@ static void setup_x11(int argc, char** argv) helper_dylibs.run_under_x11 = true; helper_dylibs.xwayland = wayland; + + // EGL is the only way to get zero-copy with dma-buf import + qputenv("QT_XCB_GL_INTEGRATION", "xcb_egl"); } } }; @@ -518,6 +521,7 @@ static void setup_opengl(bool& enable_opengl_ui) #ifndef QT_NO_OPENGL #if (defined(__arm__) || defined(__aarch64__)) && !defined(_WIN32) && !defined(__APPLE__) + // Raspberry Pi & such QSurfaceFormat fmt = QSurfaceFormat::defaultFormat(); fmt.setRenderableType(QSurfaceFormat::OpenGLES); fmt.setSwapInterval(1); @@ -533,6 +537,7 @@ static void setup_opengl(bool& enable_opengl_ui) fmt.setDefaultFormat(fmt); #else { + // Desktop GL std::vector> versions_to_test = {{4, 6}, {4, 5}, {4, 4}, {4, 3}, {4, 2}, {4, 1}, {4, 0}, {3, 3}, {3, 2}, {3, 1}, {3, 0}, {2, 1}, {2, 0}}; @@ -571,7 +576,9 @@ static void setup_opengl(bool& enable_opengl_ui) QSurfaceFormat fmt = QSurfaceFormat::defaultFormat(); fmt.setProfile(QSurfaceFormat::CoreProfile); + fmt.setRenderableType(QSurfaceFormat::OpenGL); fmt.setSwapInterval(1); + fmt.setStencilBufferSize(8); bool ok = false; for(auto [maj, min] : versions_to_test) { diff --git a/src/plugins/score-plugin-gfx/CMakeLists.txt b/src/plugins/score-plugin-gfx/CMakeLists.txt index 643785948c..04ac7c0df1 100644 --- a/src/plugins/score-plugin-gfx/CMakeLists.txt +++ b/src/plugins/score-plugin-gfx/CMakeLists.txt @@ -239,6 +239,20 @@ set(HDRS Gfx/Graph/encoders/UYVYCompute.hpp Gfx/Graph/encoders/BGRACompute.hpp Gfx/Graph/encoders/ComputeEncoder.hpp + Gfx/Graph/interop/VideoPixelFormatAV.hpp + Gfx/Graph/interop/VideoPixelFormatAV.cpp + Gfx/Graph/interop/VideoPixelFormat.hpp + Gfx/Graph/interop/V4L2PixelFormat.hpp + Gfx/Graph/interop/DirectShowPixelFormat.hpp + Gfx/Graph/interop/DrmPixelFormat.hpp + Gfx/Graph/interop/VideoPixelFormatQRhi.hpp + Gfx/Graph/interop/GStreamerPixelFormat.hpp + Gfx/Graph/interop/VideoPixelFormat.cpp + Gfx/Graph/interop/V4L2PixelFormat.cpp + Gfx/Graph/interop/DirectShowPixelFormat.cpp + Gfx/Graph/interop/DrmPixelFormat.cpp + Gfx/Graph/interop/VideoPixelFormatQRhi.cpp + Gfx/Graph/interop/GStreamerPixelFormat.cpp Gfx/Graph/decoders/GPUVideoDecoder.hpp Gfx/Graph/decoders/GPUVideoDecoderFactory.hpp diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.v4l2.cpp b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.v4l2.cpp index 9690c37aa6..6881622d80 100644 --- a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.v4l2.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.v4l2.cpp @@ -30,99 +30,63 @@ void *v4l2_mmap(void *start, size_t length, int prot, int flags, int v4l2_munmap(void *_start, size_t length); } +#include +#include + namespace Gfx { namespace { -// Imported from ffmpeg source code -struct fmt_map +// V4L2 exposes two kinds of fourcc: raw buffer layouts, and compressed +// streams. The layouts are described once, in interop/V4L2PixelFormat, shared +// with the DMA capture path -- this file used to carry a second, independently +// maintained copy of that mapping. Only the codec side stays here, because a +// codec is not a pixel format. +struct compressed_fmt { - enum AVPixelFormat ff_fmt; - enum AVCodecID codec_id; uint32_t v4l2_fmt; + AVCodecID codec_id; }; -const struct fmt_map ff_fmt_conversion_table[] = { - //ff_fmt codec_id v4l2_fmt - {AV_PIX_FMT_YUV420P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV420}, - {AV_PIX_FMT_YUV420P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YVU420}, - {AV_PIX_FMT_YUV422P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV422P}, - {AV_PIX_FMT_YUYV422, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUYV}, - {AV_PIX_FMT_UYVY422, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_UYVY}, - {AV_PIX_FMT_YUV411P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV411P}, - {AV_PIX_FMT_YUV410P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV410}, - {AV_PIX_FMT_YUV410P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YVU410}, - {AV_PIX_FMT_RGB555LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB555}, - {AV_PIX_FMT_RGB555BE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB555X}, - {AV_PIX_FMT_RGB565LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB565}, - {AV_PIX_FMT_RGB565BE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB565X}, - {AV_PIX_FMT_BGR24, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_BGR24}, - {AV_PIX_FMT_RGB24, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB24}, -#ifdef V4L2_PIX_FMT_XBGR32 - {AV_PIX_FMT_BGR0, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_XBGR32}, - {AV_PIX_FMT_0RGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_XRGB32}, - {AV_PIX_FMT_BGRA, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_ABGR32}, - {AV_PIX_FMT_ARGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_ARGB32}, -#endif - {AV_PIX_FMT_BGR0, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_BGR32}, - {AV_PIX_FMT_0RGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB32}, - {AV_PIX_FMT_GRAY8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_GREY}, -#ifdef V4L2_PIX_FMT_Y16 - {AV_PIX_FMT_GRAY16LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_Y16}, -#endif -#ifdef V4L2_PIX_FMT_Z16 - {AV_PIX_FMT_GRAY16LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_Z16}, -#endif - {AV_PIX_FMT_NV12, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_NV12}, - {AV_PIX_FMT_NONE, AV_CODEC_ID_MJPEG, V4L2_PIX_FMT_MJPEG}, - {AV_PIX_FMT_NONE, AV_CODEC_ID_MJPEG, V4L2_PIX_FMT_JPEG}, +const compressed_fmt compressed_formats[] = { + {V4L2_PIX_FMT_MJPEG, AV_CODEC_ID_MJPEG}, + {V4L2_PIX_FMT_JPEG, AV_CODEC_ID_MJPEG}, #ifdef V4L2_PIX_FMT_H264 - {AV_PIX_FMT_NONE, AV_CODEC_ID_H264, V4L2_PIX_FMT_H264}, + {V4L2_PIX_FMT_H264, AV_CODEC_ID_H264}, #endif #ifdef V4L2_PIX_FMT_MPEG4 - {AV_PIX_FMT_NONE, AV_CODEC_ID_MPEG4, V4L2_PIX_FMT_MPEG4}, + {V4L2_PIX_FMT_MPEG4, AV_CODEC_ID_MPEG4}, #endif #ifdef V4L2_PIX_FMT_CPIA1 - {AV_PIX_FMT_NONE, AV_CODEC_ID_CPIA, V4L2_PIX_FMT_CPIA1}, -#endif -#ifdef V4L2_PIX_FMT_SRGGB8 - {AV_PIX_FMT_BAYER_BGGR8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SBGGR8}, - {AV_PIX_FMT_BAYER_GBRG8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SGBRG8}, - {AV_PIX_FMT_BAYER_GRBG8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SGRBG8}, - {AV_PIX_FMT_BAYER_RGGB8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SRGGB8}, + {V4L2_PIX_FMT_CPIA1, AV_CODEC_ID_CPIA}, #endif - {AV_PIX_FMT_NONE, AV_CODEC_ID_NONE, 0}, }; -enum AVPixelFormat ff_fmt_v4l2ff(uint32_t v4l2_fmt, enum AVCodecID codec_id) +AVCodecID ff_fmt_v4l2codec(uint32_t v4l2_fmt) { - int i; - - for(i = 0; ff_fmt_conversion_table[i].codec_id != AV_CODEC_ID_NONE; i++) - { - if(ff_fmt_conversion_table[i].v4l2_fmt == v4l2_fmt - && ff_fmt_conversion_table[i].codec_id == codec_id) - { - return ff_fmt_conversion_table[i].ff_fmt; - } - } - - return AV_PIX_FMT_NONE; + for(const auto& c : compressed_formats) + if(c.v4l2_fmt == v4l2_fmt) + return c.codec_id; + return score::gfx::interop::fromV4L2PixelFormat(v4l2_fmt) + != score::gfx::interop::VideoPixelFormat::Unknown + ? AV_CODEC_ID_RAWVIDEO + : AV_CODEC_ID_NONE; } -enum AVCodecID ff_fmt_v4l2codec(uint32_t v4l2_fmt) +AVPixelFormat ff_fmt_v4l2ff(uint32_t v4l2_fmt, AVCodecID codec_id) { - int i; - - for(i = 0; ff_fmt_conversion_table[i].codec_id != AV_CODEC_ID_NONE; i++) - { - if(ff_fmt_conversion_table[i].v4l2_fmt == v4l2_fmt) - { - return ff_fmt_conversion_table[i].codec_id; - } - } - - return AV_CODEC_ID_NONE; + if(codec_id != AV_CODEC_ID_RAWVIDEO) + return AV_PIX_FMT_NONE; + + using namespace score::gfx::interop; + const auto layout = fromV4L2PixelFormat(v4l2_fmt); + if(const auto av = toAVPixelFormat(layout); av != AV_PIX_FMT_NONE) + return av; + // The V-before-U layouts have no AVPixelFormat of their own. Name the twin so + // the format stays offered, as it was before this mapping was shared; a + // consumer wanting correct chroma must exchange the U and V planes, which + // chromaSwappedTwin() is what records. + return toAVPixelFormat(chromaSwappedTwin(layout)); } class libv4l2 diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.win32.cpp b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.win32.cpp index 725a13bf26..6c4071d037 100644 --- a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.win32.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.win32.cpp @@ -17,6 +17,10 @@ extern "C" { #include #include +#include +#include +#include + namespace Gfx { static constexpr const GUID MEDIASUBTYPE_Y800 = { 0x30303859, 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}}; @@ -38,8 +42,34 @@ static constexpr const GUID MEDIASUBTYPE_Y210 = { '012Y', 0x0000, 0x static constexpr const GUID MEDIASUBTYPE_Y216 = { '612Y', 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 }}; static constexpr const GUID MEDIASUBTYPE_P408 = { '804P', 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 }}; +// GUID and score::gfx::interop::DirectShowGuid have the same layout; the +// decisions themselves live in DirectShowSubtype.hpp so they can be tested on +// hosts where this file cannot be compiled. +static const score::gfx::interop::DirectShowGuid& asPortableGuid( + const GUID& subtype) noexcept +{ + static_assert(sizeof(GUID) == sizeof(score::gfx::interop::DirectShowGuid)); + return reinterpret_cast(subtype); +} + +// True when the subtype names a compressed stream. Previously this was inferred +// from guidToPixelFormat happening to answer AV_PIX_FMT_YUVJ420P, which meant a +// camera genuinely offering that layout would have been taken for MJPEG. +static bool isCompressedSubtype(const GUID& subtype) noexcept +{ + return score::gfx::interop::directShowSubtypeIsCompressed( + asPortableGuid(subtype)); +} + static int guidToPixelFormat(const GUID& subtype) { + // The YUV layouts come from the shared, host-testable fourcc table, so they + // cannot drift and are not carrying the chroma swaps this function used to. + if(const auto av = score::gfx::interop::directShowSubtypePixelFormat( + asPortableGuid(subtype)); + av != AV_PIX_FMT_NONE) + return av; + if(subtype == MEDIASUBTYPE_RGB24) return AV_PIX_FMT_BGR24; else if(subtype == MEDIASUBTYPE_RGB32) @@ -51,6 +81,8 @@ static int guidToPixelFormat(const GUID& subtype) else if(subtype == MEDIASUBTYPE_RGB555) return AV_PIX_FMT_BGR555LE; else if(subtype == MEDIASUBTYPE_ARGB1555) + // FFmpeg has no 5:5:5 format carrying alpha, so the single alpha bit is + // dropped. The layout is otherwise identical to RGB555. return AV_PIX_FMT_BGR555LE; else if(subtype == MEDIASUBTYPE_RGB8) return AV_PIX_FMT_PAL8; @@ -86,8 +118,6 @@ static int guidToPixelFormat(const GUID& subtype) return AV_PIX_FMT_YUV420P; else if(subtype == MEDIASUBTYPE_I420) return AV_PIX_FMT_YUV420P; - else if(subtype == MEDIASUBTYPE_YV12) - return AV_PIX_FMT_YUV420P; else if(subtype == MEDIASUBTYPE_NV12) return AV_PIX_FMT_NV12; // else if(subtype == MEDIASUBTYPE_NV21) @@ -114,8 +144,6 @@ static int guidToPixelFormat(const GUID& subtype) return AV_PIX_FMT_Y210LE; else if(subtype == MEDIASUBTYPE_Y216) return AV_PIX_FMT_Y216LE; - else if(subtype == MEDIASUBTYPE_V216) // not sure - return AV_PIX_FMT_Y216LE; ///// Not supported in dshow as of ffmpeg 8, causes hangs in avformat_find_stream_info else if(subtype == MEDIASUBTYPE_Y8) { @@ -130,7 +158,7 @@ static int guidToPixelFormat(const GUID& subtype) else if(subtype == MEDIASUBTYPE_MICROSOFT_IR8) { return AV_PIX_FMT_GRAY8; } - else if (subtype == MEDIASUBTYPE_YVU9 || subtype == MEDIASUBTYPE_IF09) + else if(subtype == MEDIASUBTYPE_IF09) return AV_PIX_FMT_YUV410P; else if (subtype == MEDIASUBTYPE_Y411 || subtype == MEDIASUBTYPE_Y41P) @@ -144,8 +172,6 @@ static int guidToPixelFormat(const GUID& subtype) return AV_PIX_FMT_YUVJ420P; else if(subtype == MEDIASUBTYPE_Plum) return AV_PIX_FMT_YUVJ420P; - else if(subtype == MEDIASUBTYPE_YVU9) - return AV_PIX_FMT_YUV410P; else return AV_PIX_FMT_NONE; } @@ -163,7 +189,8 @@ static void enumerateCameraFormat( source.pixelformat = guidToPixelFormat(subtype); // Basic validation - if(width <= 0 || height <= 0 || source.pixelformat == AV_PIX_FMT_NONE) + if(width <= 0 || height <= 0 + || (source.pixelformat == AV_PIX_FMT_NONE && !isCompressedSubtype(subtype))) { // OLECHAR* guidString; // StringFromCLSID(subtype, &guidString); @@ -172,10 +199,10 @@ static void enumerateCameraFormat( return; } - // MJPEG special case - if(source.pixelformat == AV_PIX_FMT_YUVJ420P) + if(isCompressedSubtype(subtype)) { - source.codec = AV_CODEC_ID_MJPEG; + source.codec + = score::gfx::interop::directShowSubtypeCodec(asPortableGuid(subtype)); source.pixelformat = AV_PIX_FMT_NONE; } diff --git a/src/plugins/score-plugin-gfx/Gfx/GStreamer/GStreamerDevice.cpp b/src/plugins/score-plugin-gfx/Gfx/GStreamer/GStreamerDevice.cpp index 0c7e9e2e0d..93d6af325f 100644 --- a/src/plugins/score-plugin-gfx/Gfx/GStreamer/GStreamerDevice.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/GStreamer/GStreamerDevice.cpp @@ -35,8 +35,10 @@ extern "C" { #include #include