diff --git a/3rdparty/libossia b/3rdparty/libossia index 639506c5db..c2d9e3a75d 160000 --- a/3rdparty/libossia +++ b/3rdparty/libossia @@ -1 +1 @@ -Subproject commit 639506c5db56117bbfd6f83e2507c8b54ce84f93 +Subproject commit c2d9e3a75d88ed9f5202077d872cbf5bc65cec57 diff --git a/src/lib/score/tools/File.cpp b/src/lib/score/tools/File.cpp index 8941371474..333715b46a 100644 --- a/src/lib/score/tools/File.cpp +++ b/src/lib/score/tools/File.cpp @@ -110,6 +110,11 @@ locateFilePath(const QString& filename, const score::DocumentContext& ctx) noexc QString relativizeFilePath(const QString& filename, const score::DocumentContext& ctx) noexcept { + // Streamed browser files are addressed by an opaque weblocalfile: URL, not a + // filesystem path: keep it verbatim so the decoders can reopen it. + if(filename.startsWith(QLatin1String("weblocalfile:"))) + return filename; + const QFileInfo info{filename}; if(!info.isAbsolute()) diff --git a/src/plugins/score-lib-process/Process/Drop/ProcessDropHandler.cpp b/src/plugins/score-lib-process/Process/Drop/ProcessDropHandler.cpp index 7a49865840..8954d978d5 100644 --- a/src/plugins/score-lib-process/Process/Drop/ProcessDropHandler.cpp +++ b/src/plugins/score-lib-process/Process/Drop/ProcessDropHandler.cpp @@ -129,6 +129,19 @@ std::vector ProcessDropHandlerList::getDrop( if(!src.isEmpty()) { + // Large browser files are streamed on demand (custom AVIOContext over a + // QFile on the weblocalfile: Blob) instead of being copied into MEMFS, + // which is capped by the 2GB heap. Keep the weblocalfile: URL as-is so + // the decoders open it directly. /qt/tmp files can't be streamed (Qt + // deletes them after this callback) so they always get staged. + constexpr qint64 stream_threshold = 48ll * 1024 * 1024; + if(url.scheme() == QLatin1String("weblocalfile") + && QFileInfo{src}.size() > stream_threshold) + { + newUrls.push_back(url); + continue; + } + if(QFile f{src}; f.open(QIODevice::ReadOnly)) { const QByteArray bytes = f.readAll(); @@ -191,6 +204,12 @@ std::vector ProcessDropHandlerList::getDrop( for(const auto& url : mime.urls()) { auto path = url.toLocalFile(); +#if defined(__EMSCRIPTEN__) + // Un-staged large browser files keep their weblocalfile: URL (empty + // toLocalFile()); QWasmFileEngine still resolves them for QFileInfo. + if(path.isEmpty() && url.scheme() == QLatin1String("weblocalfile")) + path = url.toString(); +#endif QFileInfo f{path}; if(f.exists()) diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.cpp b/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.cpp index 0f8f8c0382..39393e6bbf 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.cpp @@ -502,7 +502,11 @@ bool DirectVideoNodeRenderer::openFile(score::gfx::GraphicsApi api, QRhi* rhi) if(m_filePath.empty()) return false; - if(avformat_open_input(&m_formatContext, m_filePath.c_str(), nullptr, nullptr) != 0) + if(auto hook = ossia::libav_custom_open()) + m_formatContext = hook(m_filePath.c_str(), m_ioOwner, m_ioFree); + + if(!m_formatContext + && avformat_open_input(&m_formatContext, m_filePath.c_str(), nullptr, nullptr) != 0) return false; if(avformat_find_stream_info(m_formatContext, nullptr) < 0) @@ -768,6 +772,12 @@ void DirectVideoNodeRenderer::closeFile() avformat_close_input(&m_formatContext); m_formatContext = nullptr; } + if(m_ioOwner) + { + m_ioFree(m_ioOwner); + m_ioOwner = nullptr; + m_ioFree = nullptr; + } m_avstream = nullptr; m_hwPixelFormat = AV_PIX_FMT_NONE; diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.hpp b/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.hpp index 3c0e766f5c..53ccb3b0c2 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.hpp @@ -101,6 +101,8 @@ class DirectVideoNodeRenderer : public NodeRenderer // Own LibAV context AVFormatContext* m_formatContext{}; + void* m_ioOwner{}; + void (*m_ioFree)(void*){}; AVCodecContext* m_codecContext{}; const void* m_codec{}; // AVCodec* AVStream* m_avstream{}; diff --git a/src/plugins/score-plugin-gfx/Gfx/Video/View.cpp b/src/plugins/score-plugin-gfx/Gfx/Video/View.cpp index 90d619ff5c..1d6f34b9a9 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Video/View.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Video/View.cpp @@ -75,6 +75,16 @@ void View::onPathChanged(const QString& str) } m_images.clear(); + m_thumb = nullptr; + +#if defined(__EMSCRIPTEN__) + // A weblocalfile: source is a browser Blob whose JS handle is bound to the + // main thread; the thumbnailer decodes on a worker thread, where reading it + // is undefined behaviour. Skip thumbnails for such (typically very large, + // streamed) sources rather than re-scanning them off-thread. + if(str.startsWith(QLatin1String("weblocalfile:"))) + return; +#endif m_thumb = new ::Video::VideoThumbnailer{str}; if(oldThread) diff --git a/src/plugins/score-plugin-media/CMakeLists.txt b/src/plugins/score-plugin-media/CMakeLists.txt index 9019a0113b..d05936735a 100644 --- a/src/plugins/score-plugin-media/CMakeLists.txt +++ b/src/plugins/score-plugin-media/CMakeLists.txt @@ -65,6 +65,7 @@ set(HDRS ${HDRS} "${CMAKE_CURRENT_SOURCE_DIR}/Media/Metro/MetroView.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/AudioDecoder.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/AvStreamIO.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/MediaFileHandle.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/LibavMediaInfo.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/RMSData.hpp" @@ -128,6 +129,7 @@ set(SRCS "${CMAKE_CURRENT_SOURCE_DIR}/Media/SndfileDecoder.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Tempo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/AudioDecoder.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/AvStreamIO.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Mixer/MixerPanel.cpp" @@ -161,6 +163,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC ### FFMPEG ### if(EMSCRIPTEN) target_include_directories(${PROJECT_NAME} PUBLIC ${OSSIA_SDK}/ffmpeg/include) + # AvStreamIO streams browser Blobs through Qt's private wasm file engine. + target_link_libraries(${PROJECT_NAME} PRIVATE ${QT_PREFIX}::CorePrivate) target_link_libraries(${PROJECT_NAME} PUBLIC -Wl,--start-group ${OSSIA_SDK}/ffmpeg/lib/libavcodec.a diff --git a/src/plugins/score-plugin-media/Media/AudioDecoder.cpp b/src/plugins/score-plugin-media/Media/AudioDecoder.cpp index cab0fac94e..28f6d364f6 100644 --- a/src/plugins/score-plugin-media/Media/AudioDecoder.cpp +++ b/src/plugins/score-plugin-media/Media/AudioDecoder.cpp @@ -1,5 +1,6 @@ #include "AudioDecoder.hpp" +#include #include #include @@ -416,19 +417,28 @@ using AVFormatContext_ptr = std::unique_ptr; using AVFrame_ptr = std::unique_ptr; -AVFormatContext_ptr open_audio(const QString& path) +AVFormatContext_ptr open_audio(const QString& path, AvIoDevice& io) { #if SCORE_HAS_LIBAV AVFormatContext* fmt_ctx_ptr{}; - auto l1 = path.toUtf8(); - auto ret = avformat_open_input(&fmt_ctx_ptr, l1.constData(), nullptr, nullptr); - if(ret != 0) + if(isStreamedMediaPath(path)) { - char err[100]{0}; - av_make_error_string(err, 100, ret); - throw std::runtime_error( - "Couldn't open file: " + std::string(l1.constData()) + " => " - + std::string(err)); + if(!open_input_custom_io(fmt_ctx_ptr, io, path)) + throw std::runtime_error( + "Couldn't open file: " + path.toStdString() + " (custom IO)"); + } + else + { + auto l1 = path.toUtf8(); + auto ret = avformat_open_input(&fmt_ctx_ptr, l1.constData(), nullptr, nullptr); + if(ret != 0) + { + char err[100]{0}; + av_make_error_string(err, 100, ret); + throw std::runtime_error( + "Couldn't open file: " + std::string(l1.constData()) + " => " + + std::string(err)); + } } AVDictionaryEntry* tag = NULL; @@ -449,7 +459,8 @@ std::optional AudioDecoder::do_probe(const QString& path) try { #if SCORE_HAS_LIBAV - auto fmt_ctx = open_audio(path); + AvIoDevice io; + auto fmt_ctx = open_audio(path, io); if(avformat_find_stream_info(fmt_ctx.get(), nullptr) < 0) return {}; @@ -707,7 +718,8 @@ void AudioDecoder::on_startDecode(QString path, audio_handle hdl) { const std::size_t channels = data.size(); - auto fmt_ctx = open_audio(path); + AvIoDevice io; + auto fmt_ctx = open_audio(path, io); auto ret = avformat_find_stream_info(fmt_ctx.get(), nullptr); if(ret != 0) diff --git a/src/plugins/score-plugin-media/Media/AvStreamIO.cpp b/src/plugins/score-plugin-media/Media/AvStreamIO.cpp new file mode 100644 index 0000000000..d634b79b1b --- /dev/null +++ b/src/plugins/score-plugin-media/Media/AvStreamIO.cpp @@ -0,0 +1,347 @@ +#include "AvStreamIO.hpp" + +#if SCORE_HAS_LIBAV +#include + +#include + +#include +#include +#include +#include +#include + +#if defined(__EMSCRIPTEN__) +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#endif + +namespace Media +{ +// ffmpeg pulls 4 MB per read, served from a 16 MB read-ahead window so +// sequential demuxing round-trips to the main thread once per 16 MB. +static constexpr int av_io_buffer_size = 1 << 22; +static constexpr int64_t prefetch_size = 16ll << 20; + +struct AvStreamState +{ + std::string url; + int64_t pos{}; + int64_t size{}; + std::vector cache; + int64_t cache_off{-1}; + int64_t cache_len{}; +}; + +bool isStreamedMediaPath(const QString& path) noexcept +{ + return path.startsWith(QLatin1String("weblocalfile:")); +} + +bool isStreamedMediaPath(const std::string& path) noexcept +{ + return path.rfind("weblocalfile:", 0) == 0; +} + +#if defined(__EMSCRIPTEN__) + +// getFile() resolves against a process-wide singleton internally, so any handler +// instance returns the File registered at drop time. +static qstdweb::File getWebFile(const std::string& url) +{ + static QWasmFileEngineHandler handler; + return handler.getFile(QString::fromStdString(url)); +} + +// Blocking read on the main thread: the Blob read suspends via JSPI, which is +// legal here because the main event loop is a promising call stack. +static int read_blob_main(const std::string& url, int64_t off, int32_t want, uint8_t* out) +{ + qstdweb::File f = getWebFile(url); + if(f.file().isUndefined() || f.file().isNull()) + return -1; + + qstdweb::ArrayBuffer ab + = f.slice((uint64_t)off, (uint64_t)(off + want)).arrayBuffer_sync(); + const int got = (int)ab.byteLength(); + if(got > 0) + qstdweb::Uint8Array(ab).copyTo((char*)out); + return got; +} + +// A worker cannot touch the main-thread Blob val, and cannot suspend via JSPI. +// So it proxies an *async* read to the main thread and blocks on a futex: the +// main job kicks off Blob.arrayBuffer() (no suspend) and its .then copies the +// bytes into shared memory and wakes the worker. `state` must stay first so its +// address is the futex word. +struct AsyncRead +{ + int32_t state{}; // 0 = pending, 1 = complete + int32_t result{}; // bytes read, or -1 on error + const std::string* url{}; + int64_t off{}; + int32_t want{}; + uint8_t* dst{}; +}; + +static void async_read_job(void* p) +{ + auto* a = static_cast(p); + qstdweb::File f = getWebFile(*a->url); + if(f.file().isUndefined() || f.file().isNull()) + { + a->result = -1; + emscripten_atomic_store_u32(&a->state, 1); + emscripten_futex_wake(&a->state, 1); + return; + } + + qstdweb::Blob blob = f.slice((uint64_t)a->off, (uint64_t)(a->off + a->want)); + EM_ASM( + { + var blob = Emval.toValue($0); + var dst = $1; + var statePtr = $2; + var resultPtr = $3; + blob.arrayBuffer() + .then(function(ab) { + var u8 = new Uint8Array(ab); + HEAPU8.set(u8, dst); + HEAP32[resultPtr >> 2] = u8.length; + Atomics.store(HEAP32, statePtr >> 2, 1); + Atomics.notify(HEAP32, statePtr >> 2); + }) + .catch(function(e) { + HEAP32[resultPtr >> 2] = -1; + Atomics.store(HEAP32, statePtr >> 2, 1); + Atomics.notify(HEAP32, statePtr >> 2); + }); + }, + blob.val().as_handle(), a->dst, &a->state, &a->result); +} + +static int read_blob_worker(const std::string& url, int64_t off, int32_t want, uint8_t* out) +{ + AsyncRead a; + a.url = &url; + a.off = off; + a.want = want; + a.dst = out; + + if(!emscripten_proxy_async( + emscripten_proxy_get_system_queue(), emscripten_main_runtime_thread_id(), + &async_read_job, &a)) + return -1; + + while(emscripten_atomic_load_u32(&a.state) == 0) + emscripten_futex_wait(&a.state, 0, INFINITY); + + return a.result; +} + +static int fetch_bytes(const std::string& url, int64_t off, int32_t want, uint8_t* dst) +{ + return emscripten_is_main_runtime_thread() + ? read_blob_main(url, off, want, dst) + : read_blob_worker(url, off, want, dst); +} + +static int av_io_read(void* opaque, uint8_t* out, int size) +{ + auto* st = static_cast(opaque); + const int64_t avail = st->size - st->pos; + if(avail <= 0) + return AVERROR_EOF; + + const int32_t want = (int32_t)std::min(size, avail); + + if(st->cache_off >= 0 && st->pos >= st->cache_off + && st->pos + want <= st->cache_off + st->cache_len) + { + std::memcpy(out, st->cache.data() + (st->pos - st->cache_off), want); + st->pos += want; + return want; + } + + const int32_t win = (int32_t)std::min(prefetch_size, st->size - st->pos); + const int got = fetch_bytes(st->url, st->pos, win, st->cache.data()); + if(got <= 0) + return got == 0 ? AVERROR_EOF : AVERROR(EIO); + st->cache_off = st->pos; + st->cache_len = got; + + const int n = std::min(want, got); + std::memcpy(out, st->cache.data(), n); + st->pos += n; + return n; +} + +static int64_t av_io_seek(void* opaque, int64_t offset, int whence) +{ + auto* st = static_cast(opaque); + switch(whence & ~AVSEEK_FORCE) + { + case AVSEEK_SIZE: + return st->size; + case SEEK_SET: + st->pos = offset; + break; + case SEEK_CUR: + st->pos += offset; + break; + case SEEK_END: + st->pos = st->size + offset; + break; + default: + return -1; + } + if(st->pos < 0) + st->pos = 0; + return st->pos; +} + +#endif + +AvIoDevice::AvIoDevice(const QString& path) +{ + auto st = std::make_unique(); + st->url = path.toStdString(); + +#if defined(__EMSCRIPTEN__) + bool ok = false; + const auto fetch_size = [&] { + qstdweb::File f = getWebFile(st->url); + if(f.file().isUndefined() || f.file().isNull()) + return; + st->size = (int64_t)f.size(); + ok = st->size > 0; + }; + if(emscripten_is_main_runtime_thread()) + { + fetch_size(); + } + else + { + emscripten_proxy_sync( + emscripten_proxy_get_system_queue(), emscripten_main_runtime_thread_id(), + [](void* p) { (*static_cast(p))(); }, + const_cast(static_cast(&fetch_size))); + } + if(!ok) + return; + + st->cache.resize((size_t)std::min(prefetch_size, st->size)); + + auto* buffer = static_cast(av_malloc(av_io_buffer_size)); + if(!buffer) + return; + + avio = avio_alloc_context( + buffer, av_io_buffer_size, 0, st.get(), &av_io_read, nullptr, &av_io_seek); + if(!avio) + { + av_free(buffer); + return; + } + state = st.release(); +#endif +} + +AvIoDevice::~AvIoDevice() +{ + if(avio) + { + av_freep(&avio->buffer); + avio_context_free(&avio); + } + delete state; + state = nullptr; +} + +AvIoDevice::AvIoDevice(AvIoDevice&& other) noexcept + : avio{other.avio} + , state{other.state} +{ + other.avio = nullptr; + other.state = nullptr; +} + +AvIoDevice& AvIoDevice::operator=(AvIoDevice&& other) noexcept +{ + if(this != &other) + { + if(avio) + { + av_freep(&avio->buffer); + avio_context_free(&avio); + } + delete state; + avio = other.avio; + state = other.state; + other.avio = nullptr; + other.state = nullptr; + } + return *this; +} + +bool open_input_custom_io( + AVFormatContext*& format, AvIoDevice& io, const QString& path) noexcept +{ + io = AvIoDevice{path}; + if(!io) + return false; + + AVFormatContext* fmt = avformat_alloc_context(); + if(!fmt) + { + io = {}; + return false; + } + + fmt->pb = io.avio; + fmt->flags |= AVFMT_FLAG_CUSTOM_IO; + + if(avformat_open_input(&fmt, nullptr, nullptr, nullptr) != 0) + { + io = {}; + return false; + } + + format = fmt; + return true; +} + +// libossia's libav_handle is Qt-agnostic; it delegates the browser-only Blob +// streaming back here through a hook so the submodule keeps no Qt dependency. +static AVFormatContext* libav_stream_open_hook( + const char* path, void*& io_owner, void (*&io_free)(void*)) noexcept +{ + const QString qpath = QString::fromUtf8(path); + if(!isStreamedMediaPath(qpath)) + return nullptr; + + auto io = std::make_unique(); + AVFormatContext* fmt{}; + if(!open_input_custom_io(fmt, *io, qpath)) + return nullptr; + + io_owner = io.release(); + io_free = [](void* p) { delete static_cast(p); }; + return fmt; +} + +static const bool registered_libav_hook = [] { + ossia::libav_custom_open() = &libav_stream_open_hook; + return true; +}(); +} +#endif diff --git a/src/plugins/score-plugin-media/Media/AvStreamIO.hpp b/src/plugins/score-plugin-media/Media/AvStreamIO.hpp new file mode 100644 index 0000000000..85bd74d62a --- /dev/null +++ b/src/plugins/score-plugin-media/Media/AvStreamIO.hpp @@ -0,0 +1,57 @@ +#pragma once +#include + +#if SCORE_HAS_LIBAV +extern "C" { +#include +} + +#include + +#include + +class QString; + +namespace Media +{ +struct AvStreamState; + +// A path that must be streamed on demand rather than opened with fopen/avio's +// default file protocol. On the wasm build these are the "weblocalfile:/N/name" +// URLs Qt assigns to browser File objects: fopen cannot open them, but the +// backing Blob can be read in byte-ranges, so a multi-GB file never has to be +// copied into MEMFS/RAM. +SCORE_PLUGIN_MEDIA_EXPORT +bool isStreamedMediaPath(const QString& path) noexcept; +SCORE_PLUGIN_MEDIA_EXPORT +bool isStreamedMediaPath(const std::string& path) noexcept; + +// Owns the AVIOContext that streams a browser Blob and the position bookkeeping +// it reads through. Must outlive the AVFormatContext it is attached to (see +// open_input_custom_io). +struct SCORE_PLUGIN_MEDIA_EXPORT AvIoDevice +{ + AvIoDevice() = default; + explicit AvIoDevice(const QString& path); + ~AvIoDevice(); + + AvIoDevice(AvIoDevice&&) noexcept; + AvIoDevice& operator=(AvIoDevice&&) noexcept; + AvIoDevice(const AvIoDevice&) = delete; + AvIoDevice& operator=(const AvIoDevice&) = delete; + + explicit operator bool() const noexcept { return avio != nullptr; } + + AVIOContext* avio{}; + AvStreamState* state{}; +}; + +// Allocates `format` and opens `path` through a custom AVIOContext backed by a +// QFile. On success `format` is set and `io` owns the streaming resources; the +// caller must destroy `format` (avformat_close_input) before `io`. +// Returns false and leaves `format` null on failure. +SCORE_PLUGIN_MEDIA_EXPORT +bool open_input_custom_io( + AVFormatContext*& format, AvIoDevice& io, const QString& path) noexcept; +} +#endif diff --git a/src/plugins/score-plugin-media/Media/LibavMediaInfo.cpp b/src/plugins/score-plugin-media/Media/LibavMediaInfo.cpp index 1d857bf232..7ea83710f1 100644 --- a/src/plugins/score-plugin-media/Media/LibavMediaInfo.cpp +++ b/src/plugins/score-plugin-media/Media/LibavMediaInfo.cpp @@ -2,6 +2,8 @@ #if SCORE_HAS_LIBAV +#include + #include #include @@ -17,21 +19,47 @@ bool has_valid_duration(const AVFormatContext* ctx) noexcept && ctx->duration != std::numeric_limits::min() && ctx->duration > 0; } -MediaInfo::FormatContextPtr try_open(const char* path, AVDictionary** opts) +struct OpenedInput { - AVFormatContext* raw = avformat_alloc_context(); - if(!raw) - return {}; + MediaInfo::FormatContextPtr ctx; + std::shared_ptr io; + explicit operator bool() const noexcept { return static_cast(ctx); } +}; - if(avformat_open_input(&raw, path, nullptr, opts) != 0) - return {}; +OpenedInput try_open(const QString& path, AVDictionary** opts) +{ + std::shared_ptr io; + AVFormatContext* raw{}; + + if(Media::isStreamedMediaPath(path)) + { + auto dev = std::make_shared(path); + if(!*dev) + return {}; + raw = avformat_alloc_context(); + if(!raw) + return {}; + raw->pb = dev->avio; + raw->flags |= AVFMT_FLAG_CUSTOM_IO; + if(avformat_open_input(&raw, nullptr, nullptr, opts) != 0) + return {}; + io = std::move(dev); + } + else + { + raw = avformat_alloc_context(); + if(!raw) + return {}; + if(avformat_open_input(&raw, path.toUtf8().constData(), nullptr, opts) != 0) + return {}; + } MediaInfo::FormatContextPtr ctx{raw}; if(avformat_find_stream_info(ctx.get(), nullptr) < 0 || ctx->nb_streams == 0) return {}; - return ctx; + return {std::move(ctx), std::move(io)}; } // Walk every packet to compute duration manually. Works on any file whose @@ -102,14 +130,15 @@ int64_t scan_duration_from_packets(AVFormatContext* ctx) noexcept return AV_NOPTS_VALUE; } -MediaInfo build_info(MediaInfo::FormatContextPtr ctx, std::optional duration_av) +MediaInfo build_info(OpenedInput opened, std::optional duration_av) { MediaInfo info; info.duration_av = duration_av; - info.streams.reserve(ctx->nb_streams); - for(unsigned i = 0; i < ctx->nb_streams; i++) - info.streams.push_back(ctx->streams[i]->codecpar->codec_type); - info.format_context = std::move(ctx); + info.streams.reserve(opened.ctx->nb_streams); + for(unsigned i = 0; i < opened.ctx->nb_streams; i++) + info.streams.push_back(opened.ctx->streams[i]->codecpar->codec_type); + info.io_backing = std::move(opened.io); + info.format_context = std::move(opened.ctx); return info; } @@ -117,16 +146,13 @@ MediaInfo build_info(MediaInfo::FormatContextPtr ctx, std::optional dur std::optional probe(const QString& path) { - const QByteArray utf8 = path.toUtf8(); - const char* cpath = utf8.constData(); - // ---- Attempt 1: defaults (fast path) ---- - if(auto ctx = try_open(cpath, nullptr)) + if(auto opened = try_open(path, nullptr)) { - if(has_valid_duration(ctx.get())) + if(has_valid_duration(opened.ctx.get())) { - const int64_t dur = ctx->duration; - return build_info(std::move(ctx), dur); + const int64_t dur = opened.ctx->duration; + return build_info(std::move(opened), dur); } // else: fall through, but drop this context first } @@ -137,13 +163,13 @@ std::optional probe(const QString& path) av_dict_set(&opts, "probesize", "100000000", 0); av_dict_set(&opts, "analyzeduration", "100000000", 0); - auto ctx = try_open(cpath, &opts); + auto opened = try_open(path, &opts); av_dict_free(&opts); - if(ctx && has_valid_duration(ctx.get())) + if(opened && has_valid_duration(opened.ctx.get())) { - const int64_t dur = ctx->duration; - return build_info(std::move(ctx), dur); + const int64_t dur = opened.ctx->duration; + return build_info(std::move(opened), dur); } } @@ -154,24 +180,24 @@ std::optional probe(const QString& path) av_dict_set(&opts, "analyzeduration", "100000000", 0); av_dict_set(&opts, "fflags", "+ignidx+genpts", 0); - auto ctx = try_open(cpath, &opts); + auto opened = try_open(path, &opts); av_dict_free(&opts); - if(ctx) + if(opened) { - if(has_valid_duration(ctx.get())) + if(has_valid_duration(opened.ctx.get())) { - const int64_t dur = ctx->duration; - return build_info(std::move(ctx), dur); + const int64_t dur = opened.ctx->duration; + return build_info(std::move(opened), dur); } // ---- Attempt 4: full packet scan on the same context ---- - int64_t scanned = scan_duration_from_packets(ctx.get()); + int64_t scanned = scan_duration_from_packets(opened.ctx.get()); if(scanned != AV_NOPTS_VALUE && scanned > 0) - return build_info(std::move(ctx), scanned); + return build_info(std::move(opened), scanned); // File is structurally readable but no duration could be determined. - return build_info(std::move(ctx), std::nullopt); + return build_info(std::move(opened), std::nullopt); } } diff --git a/src/plugins/score-plugin-media/Media/LibavMediaInfo.hpp b/src/plugins/score-plugin-media/Media/LibavMediaInfo.hpp index 1b688c71bd..78028ff284 100644 --- a/src/plugins/score-plugin-media/Media/LibavMediaInfo.hpp +++ b/src/plugins/score-plugin-media/Media/LibavMediaInfo.hpp @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -42,6 +43,10 @@ struct SCORE_PLUGIN_MEDIA_EXPORT MediaInfo }; using FormatContextPtr = std::unique_ptr; + // Streaming backing (e.g. an AvIoDevice for weblocalfile: sources). Declared + // before format_context so it is destroyed *after* it: avformat_close_input + // must run while the custom AVIOContext is still alive. + std::shared_ptr io_backing; FormatContextPtr format_context; std::vector streams; std::optional duration_av; // in AV_TIME_BASE units; empty == unknown diff --git a/src/plugins/score-plugin-media/Media/Sound/Drop/SoundDrop.cpp b/src/plugins/score-plugin-media/Media/Sound/Drop/SoundDrop.cpp index b4ab5f9e5e..d1ea6bbedf 100644 --- a/src/plugins/score-plugin-media/Media/Sound/Drop/SoundDrop.cpp +++ b/src/plugins/score-plugin-media/Media/Sound/Drop/SoundDrop.cpp @@ -22,7 +22,11 @@ DroppedAudioFiles::DroppedAudioFiles( for(const auto& url : urls) { QString filename = url.toLocalFile(); - if(!(AudioFile::isSupported(QFile{filename}) +#if defined(__EMSCRIPTEN__) + if(filename.isEmpty() && url.scheme() == QLatin1String("weblocalfile")) + filename = url.toString(); +#endif + if(!(AudioFile::isSupported(QFile{filename}) || AudioFile::isSupportedVideo(QFile{filename}))) continue; diff --git a/src/plugins/score-plugin-media/Media/Sound/SoundView.cpp b/src/plugins/score-plugin-media/Media/Sound/SoundView.cpp index d2da8d94f5..bca10bcd42 100644 --- a/src/plugins/score-plugin-media/Media/Sound/SoundView.cpp +++ b/src/plugins/score-plugin-media/Media/Sound/SoundView.cpp @@ -98,6 +98,14 @@ void LayerView::recompute() const || m_model.file()->sampleRate() < 1.)) return; +#if defined(__EMSCRIPTEN__) + // The waveform is computed on a worker thread. A weblocalfile: source is a + // browser Blob bound to the main thread, so decoding it off-thread is + // undefined behaviour; skip the waveform for such (streamed) sources. + if(m_model.file()->absoluteFileName().startsWith(QLatin1String("weblocalfile:"))) + return; +#endif + if(auto view = getView(*this)) { // By default we try to force a render of everything, but it's too slow with very large files diff --git a/src/plugins/score-plugin-media/Video/Thumbnailer.cpp b/src/plugins/score-plugin-media/Video/Thumbnailer.cpp index 403ec5e244..bc0a55dd6d 100644 --- a/src/plugins/score-plugin-media/Video/Thumbnailer.cpp +++ b/src/plugins/score-plugin-media/Video/Thumbnailer.cpp @@ -32,15 +32,22 @@ VideoThumbnailer::VideoThumbnailer(QString path) Qt::QueuedConnection); auto inputFile = path.toUtf8(); - if(int err = avformat_open_input(&m_formatContext, inputFile.data(), nullptr, nullptr); - err != 0) + if(auto hook = ossia::libav_custom_open()) + m_formatContext = hook(inputFile.data(), m_ioOwner, m_ioFree); + + if(!m_formatContext) { - char str[1500]; - qDebug() << "VideoThumbnailer: avformat_open_input failed" - << av_make_error_string(str, 1499, err); + if(int err + = avformat_open_input(&m_formatContext, inputFile.data(), nullptr, nullptr); + err != 0) + { + char str[1500]; + qDebug() << "VideoThumbnailer: avformat_open_input failed" + << av_make_error_string(str, 1499, err); - SCORE_ASSERT(m_formatContext == nullptr); - return; + SCORE_ASSERT(m_formatContext == nullptr); + return; + } } if(int err = avformat_find_stream_info(m_formatContext, nullptr); err < 0) @@ -51,6 +58,12 @@ VideoThumbnailer::VideoThumbnailer(QString path) avformat_close_input(&m_formatContext); m_formatContext = nullptr; + if(m_ioOwner) + { + m_ioFree(m_ioOwner); + m_ioOwner = nullptr; + m_ioFree = nullptr; + } return; } @@ -177,6 +190,12 @@ VideoThumbnailer::~VideoThumbnailer() avformat_close_input(&m_formatContext); m_formatContext = nullptr; } + if(m_ioOwner) + { + m_ioFree(m_ioOwner); + m_ioOwner = nullptr; + m_ioFree = nullptr; + } } void VideoThumbnailer::onRequest(int64_t req, QVector flicks) diff --git a/src/plugins/score-plugin-media/Video/Thumbnailer.hpp b/src/plugins/score-plugin-media/Video/Thumbnailer.hpp index d6281c23f8..67aa0d5b0e 100644 --- a/src/plugins/score-plugin-media/Video/Thumbnailer.hpp +++ b/src/plugins/score-plugin-media/Video/Thumbnailer.hpp @@ -48,6 +48,8 @@ class SCORE_PLUGIN_MEDIA_EXPORT VideoThumbnailer int m_currentIndex{}; AVFormatContext* m_formatContext{}; + void* m_ioOwner{}; + void (*m_ioFree)(void*){}; AVCodecContext* m_codecContext{}; SwsContext* m_rescale{}; const AVCodec* m_codec{}; diff --git a/src/plugins/score-plugin-media/Video/VideoDecoder.cpp b/src/plugins/score-plugin-media/Video/VideoDecoder.cpp index a64db145f1..5fd45b71d0 100644 --- a/src/plugins/score-plugin-media/Video/VideoDecoder.cpp +++ b/src/plugins/score-plugin-media/Video/VideoDecoder.cpp @@ -362,7 +362,16 @@ bool VideoDecoder::open(const std::string& inputFile) noexcept m_inputFile = inputFile; this->filePath = inputFile; - if(avformat_open_input(&m_formatContext, inputFile.c_str(), nullptr, nullptr) != 0) + if(Media::isStreamedMediaPath(inputFile)) + { + if(!Media::open_input_custom_io( + m_formatContext, m_io, QString::fromStdString(inputFile))) + { + close_file(); + return false; + } + } + else if(avformat_open_input(&m_formatContext, inputFile.c_str(), nullptr, nullptr) != 0) { close_file(); return false; @@ -491,6 +500,7 @@ void VideoDecoder::close_file() noexcept avformat_close_input(&m_formatContext); m_formatContext = nullptr; } + m_io = Media::AvIoDevice{}; } ReadFrame LibAVDecoder::read_one_frame_raw(AVPacket& packet) diff --git a/src/plugins/score-plugin-media/Video/VideoDecoder.hpp b/src/plugins/score-plugin-media/Video/VideoDecoder.hpp index ef4d94dc67..99c6a4f0dc 100644 --- a/src/plugins/score-plugin-media/Video/VideoDecoder.hpp +++ b/src/plugins/score-plugin-media/Video/VideoDecoder.hpp @@ -1,6 +1,7 @@ #pragma once #include #if SCORE_HAS_LIBAV +#include #include