wasm: stream large media from Blob URLs instead of copying into RAM (Tier B)

src/lib/score/tools/File.cpp

@@ -110,6 +110,11 @@ locateFilePath(const QString& filename, const score::DocumentContext& ctx) noexc
 QString
 relativizeFilePath(const QString& filename, const score::DocumentContext& ctx) noexcept
 {
+  // Streamed browser files are addressed by an opaque weblocalfile: URL, not a
+  // filesystem path: keep it verbatim so the decoders can reopen it.
+  if(filename.startsWith(QLatin1String("weblocalfile:")))
+    return filename;
+
   const QFileInfo info{filename};
 
   if(!info.isAbsolute())

src/plugins/score-lib-process/Process/Drop/ProcessDropHandler.cpp

@@ -129,6 +129,19 @@ std::vector<ProcessDropHandler::ProcessDrop> ProcessDropHandlerList::getDrop(
 
       if(!src.isEmpty())
       {
+        // Large browser files are streamed on demand (custom AVIOContext over a
+        // QFile on the weblocalfile: Blob) instead of being copied into MEMFS,
+        // which is capped by the 2GB heap. Keep the weblocalfile: URL as-is so
+        // the decoders open it directly. /qt/tmp files can't be streamed (Qt
+        // deletes them after this callback) so they always get staged.
+        constexpr qint64 stream_threshold = 48ll * 1024 * 1024;
+        if(url.scheme() == QLatin1String("weblocalfile")
+           && QFileInfo{src}.size() > stream_threshold)
+        {
+          newUrls.push_back(url);
+          continue;
+        }
+
         if(QFile f{src}; f.open(QIODevice::ReadOnly))
         {
           const QByteArray bytes = f.readAll();
@@ -191,6 +204,12 @@ std::vector<ProcessDropHandler::ProcessDrop> ProcessDropHandlerList::getDrop(
   for(const auto& url : mime.urls())
   {
     auto path = url.toLocalFile();
+#if defined(__EMSCRIPTEN__)
+    // Un-staged large browser files keep their weblocalfile: URL (empty
+    // toLocalFile()); QWasmFileEngine still resolves them for QFileInfo.
+    if(path.isEmpty() && url.scheme() == QLatin1String("weblocalfile"))
+      path = url.toString();
+#endif
 
     QFileInfo f{path};
     if(f.exists())

src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.cpp

@@ -502,7 +502,11 @@ bool DirectVideoNodeRenderer::openFile(score::gfx::GraphicsApi api, QRhi* rhi)
   if(m_filePath.empty())
     return false;
 
-  if(avformat_open_input(&m_formatContext, m_filePath.c_str(), nullptr, nullptr) != 0)
+  if(auto hook = ossia::libav_custom_open())
+    m_formatContext = hook(m_filePath.c_str(), m_ioOwner, m_ioFree);
+
+  if(!m_formatContext
+     && avformat_open_input(&m_formatContext, m_filePath.c_str(), nullptr, nullptr) != 0)
     return false;
 
   if(avformat_find_stream_info(m_formatContext, nullptr) < 0)
@@ -768,6 +772,12 @@ void DirectVideoNodeRenderer::closeFile()
     avformat_close_input(&m_formatContext);
     m_formatContext = nullptr;
   }
+  if(m_ioOwner)
+  {
+    m_ioFree(m_ioOwner);
+    m_ioOwner = nullptr;
+    m_ioFree = nullptr;
+  }
 
   m_avstream = nullptr;
   m_hwPixelFormat = AV_PIX_FMT_NONE;

src/plugins/score-plugin-gfx/Gfx/Graph/DirectVideoNodeRenderer.hpp

@@ -101,6 +101,8 @@ class DirectVideoNodeRenderer : public NodeRenderer
 
   // Own LibAV context
   AVFormatContext* m_formatContext{};
+  void* m_ioOwner{};
+  void (*m_ioFree)(void*){};
   AVCodecContext* m_codecContext{};
   const void* m_codec{}; // AVCodec*
   AVStream* m_avstream{};

src/plugins/score-plugin-gfx/Gfx/Video/View.cpp

@@ -75,6 +75,16 @@ void View::onPathChanged(const QString& str)
   }
 
   m_images.clear();
+  m_thumb = nullptr;
+
+#if defined(__EMSCRIPTEN__)
+  // A weblocalfile: source is a browser Blob whose JS handle is bound to the
+  // main thread; the thumbnailer decodes on a worker thread, where reading it
+  // is undefined behaviour. Skip thumbnails for such (typically very large,
+  // streamed) sources rather than re-scanning them off-thread.
+  if(str.startsWith(QLatin1String("weblocalfile:")))
+    return;
+#endif
 
   m_thumb = new ::Video::VideoThumbnailer{str};
   if(oldThread)

src/plugins/score-plugin-media/CMakeLists.txt

@@ -65,6 +65,7 @@ set(HDRS ${HDRS}
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/Metro/MetroView.hpp"
 
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/AudioDecoder.hpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Media/AvStreamIO.hpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/MediaFileHandle.hpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/LibavMediaInfo.hpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/RMSData.hpp"
@@ -128,6 +129,7 @@ set(SRCS
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/SndfileDecoder.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/Tempo.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/Media/AudioDecoder.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Media/AvStreamIO.cpp"
 
     "${CMAKE_CURRENT_SOURCE_DIR}/Mixer/MixerPanel.cpp"
 
@@ -161,6 +163,8 @@ target_link_libraries(${PROJECT_NAME} PUBLIC
 ### FFMPEG ###
 if(EMSCRIPTEN)
   target_include_directories(${PROJECT_NAME} PUBLIC ${OSSIA_SDK}/ffmpeg/include)
+  # AvStreamIO streams browser Blobs through Qt's private wasm file engine.
+  target_link_libraries(${PROJECT_NAME} PRIVATE ${QT_PREFIX}::CorePrivate)
   target_link_libraries(${PROJECT_NAME} PUBLIC
     -Wl,--start-group
     ${OSSIA_SDK}/ffmpeg/lib/libavcodec.a

src/plugins/score-plugin-media/Media/AudioDecoder.cpp

@@ -1,5 +1,6 @@
 #include "AudioDecoder.hpp"
 
+#include <Media/AvStreamIO.hpp>
 #include <Media/Libav.hpp>
 #include <Media/Sound/SoundModel.hpp>
 
@@ -416,19 +417,28 @@ using AVFormatContext_ptr = std::unique_ptr<AVFormatContext, AVFormatContext_Fre
 using AVCodecContext_ptr = std::unique_ptr<AVCodecContext, AVCodecContext_Free>;
 using AVFrame_ptr = std::unique_ptr<AVFrame, AVFrame_Free>;
 
-AVFormatContext_ptr open_audio(const QString& path)
+AVFormatContext_ptr open_audio(const QString& path, AvIoDevice& io)
 {
 #if SCORE_HAS_LIBAV
   AVFormatContext* fmt_ctx_ptr{};
-  auto l1 = path.toUtf8();
-  auto ret = avformat_open_input(&fmt_ctx_ptr, l1.constData(), nullptr, nullptr);
-  if(ret != 0)
+  if(isStreamedMediaPath(path))
   {
-    char err[100]{0};
-    av_make_error_string(err, 100, ret);
-    throw std::runtime_error(
-        "Couldn't open file: " + std::string(l1.constData()) + " => "
-        + std::string(err));
+    if(!open_input_custom_io(fmt_ctx_ptr, io, path))
+      throw std::runtime_error(
+          "Couldn't open file: " + path.toStdString() + " (custom IO)");
+  }
+  else
+  {
+    auto l1 = path.toUtf8();
+    auto ret = avformat_open_input(&fmt_ctx_ptr, l1.constData(), nullptr, nullptr);
+    if(ret != 0)
+    {
+      char err[100]{0};
+      av_make_error_string(err, 100, ret);
+      throw std::runtime_error(
+          "Couldn't open file: " + std::string(l1.constData()) + " => "
+          + std::string(err));
+    }
   }
 
   AVDictionaryEntry* tag = NULL;
@@ -449,7 +459,8 @@ std::optional<AudioInfo> AudioDecoder::do_probe(const QString& path)
 try
 {
 #if SCORE_HAS_LIBAV
-  auto fmt_ctx = open_audio(path);
+  AvIoDevice io;
+  auto fmt_ctx = open_audio(path, io);
 
   if(avformat_find_stream_info(fmt_ctx.get(), nullptr) < 0)
     return {};
@@ -707,7 +718,8 @@ void AudioDecoder::on_startDecode(QString path, audio_handle hdl)
   {
     const std::size_t channels = data.size();
 
-    auto fmt_ctx = open_audio(path);
+    AvIoDevice io;
+    auto fmt_ctx = open_audio(path, io);
 
     auto ret = avformat_find_stream_info(fmt_ctx.get(), nullptr);
     if(ret != 0)