From b3d2e6ee73ae7595d3b0d3e56c5b04e8b597aca4 Mon Sep 17 00:00:00 2001 From: Alfredo Bautista Date: Wed, 8 Jul 2026 10:07:31 +0200 Subject: [PATCH] Add Linux/Windows support for audio renderer (audio frame capture) Implements the startAudioRenderer / stopAudioRenderer method-channel handlers on Linux and Windows, bringing desktop platforms to parity with iOS/macOS/Android for AudioFrameCapture. Previously these calls fell through to NotImplemented(), surfacing in Dart as MissingPluginException. - shared_cpp/audio_renderer.{h,cpp}: platform-agnostic conversion pipeline (validate -> resample -> channel reduce -> int16/float32 encode), a direct port of the Android AudioRenderer/AudioResampler. - linux/windows livekit_plugin.cpp: AudioRendererSink attaching to tracks via libwebrtc AudioTrackSink, streaming converted frames to Dart over io.livekit.audio.renderer/channel-. --- CHANGELOG.md | 4 + linux/CMakeLists.txt | 1 + linux/livekit_plugin.cpp | 178 ++++++++++++++++++++++++++++++++++ shared_cpp/audio_renderer.cpp | 160 ++++++++++++++++++++++++++++++ shared_cpp/audio_renderer.h | 48 +++++++++ windows/CMakeLists.txt | 1 + windows/livekit_plugin.cpp | 178 ++++++++++++++++++++++++++++++++++ 7 files changed, 570 insertions(+) create mode 100644 shared_cpp/audio_renderer.cpp create mode 100644 shared_cpp/audio_renderer.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f1778128..6e62446f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## Unreleased + +* Added: Linux/Windows support for audio renderer (audio frame capture) + ## 2.8.0 * Added: Session API support for simpler E2EE setup diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 218351f8f..dacb89dfc 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -29,6 +29,7 @@ list(APPEND PLUGIN_SOURCES "task_runner_linux.cc" "../shared_cpp/fft_processor.cpp" "../shared_cpp/audio_visualizer.cpp" + "../shared_cpp/audio_renderer.cpp" "../shared_cpp/pffft.c" "flutter/core_implementations.cc" "flutter/standard_codec.cc" diff --git a/linux/livekit_plugin.cpp b/linux/livekit_plugin.cpp index 8a635b539..4c29d5e92 100644 --- a/linux/livekit_plugin.cpp +++ b/linux/livekit_plugin.cpp @@ -29,10 +29,12 @@ #include #include #include +#include #include #include #include +#include "audio_renderer.h" #include "audio_visualizer.h" #include "task_runner_linux.h" @@ -159,6 +161,104 @@ class VisualizerSink : public libwebrtc::AudioTrackSink { int bar_count_ = 7; }; +class AudioRendererSink : public libwebrtc::AudioTrackSink { +public: + AudioRendererSink( + BinaryMessenger *messenger, std::string event_channel_name, + libwebrtc::scoped_refptr media_track, + RendererAudioFormat format) + : channel_( + std::make_unique>( + messenger, event_channel_name, + &flutter::StandardMethodCodec::GetInstance())), + media_track_(media_track), format_(format) { + task_runner_ = std::make_unique(); + auto handler = std::make_unique< + flutter::StreamHandlerFunctions>( + [&](const flutter::EncodableValue *arguments, + std::unique_ptr> + &&events) + -> std::unique_ptr< + flutter::StreamHandlerError> { + sink_ = std::move(events); + on_listen_called_ = true; + return nullptr; + }, + [&](const flutter::EncodableValue *arguments) + -> std::unique_ptr< + flutter::StreamHandlerError> { + on_listen_called_ = false; + return nullptr; + }); + + channel_->SetStreamHandler(std::move(handler)); + ((libwebrtc::RTCAudioTrack *)media_track_.get())->AddSink(this); + } + ~AudioRendererSink() override {} + +public: + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames) override { + // Unlike VisualizerSink, live audio must not be queued while no + // subscriber is attached — drop the frame instead. + if (!on_listen_called_) { + return; + } + + AudioConversionResult conversion = + ConvertAudioData(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames, format_); + if (!conversion.success) { + LogDroppedFrame(); + return; + } + + EncodableMap payload; + payload[EncodableValue("commonFormat")] = + EncodableValue(format_.common_format); + payload[EncodableValue("sampleRate")] = + EncodableValue(format_.sample_rate); + payload[EncodableValue("channels")] = EncodableValue(conversion.channels); + payload[EncodableValue("frameLength")] = + EncodableValue(conversion.frame_length); + payload[EncodableValue("data")] = EncodableValue(std::move(conversion.data)); + + PostEvent(EncodableValue(payload)); + } + + void PostEvent(const flutter::EncodableValue &event) { + std::weak_ptr> weak_sink = sink_; + task_runner_->EnqueueTask([weak_sink, event]() { + auto sink = weak_sink.lock(); + if (sink) { + sink->Success(event); + } + }); + } + + void RemoveSink() { + ((libwebrtc::RTCAudioTrack *)media_track_.get())->RemoveSink(this); + channel_->SetStreamHandler(nullptr); + } + +private: + void LogDroppedFrame() { + dropped_frame_count_ += 1; + if (dropped_frame_count_ <= 5 || dropped_frame_count_ % 100 == 0) { + std::cerr << "[LiveKit] Dropping audio frame #" << dropped_frame_count_ + << " for audio renderer" << std::endl; + } + } + + std::unique_ptr task_runner_; + std::unique_ptr> channel_; + std::shared_ptr> sink_; + bool on_listen_called_ = false; + libwebrtc::scoped_refptr media_track_; + RendererAudioFormat format_; + int64_t dropped_frame_count_ = 0; +}; + class LiveKitPlugin : public flutter::Plugin { public: static void RegisterWithRegistrar(flutter::PluginRegistrar *registrar); @@ -176,6 +276,8 @@ class LiveKitPlugin : public flutter::Plugin { private: flutter_webrtc_plugin::FlutterWebRTC *webrtc_instance_ = nullptr; std::unordered_map> visualizers_; + std::unordered_map> + renderers_; BinaryMessenger *messenger_ = nullptr; mutable std::mutex mutex_; }; @@ -274,6 +376,82 @@ void LiveKitPlugin::HandleMethodCall( } result->Success(); + } else if (method_call.method_name().compare("startAudioRenderer") == 0) { + if (!method_call.arguments()) { + result->Error("Bad Arguments", "Null arguments received"); + return; + } + flutter::EncodableMap params = + GetValue(*method_call.arguments()); + std::string trackId = findString(params, "trackId"); + std::string rendererId = findString(params, "rendererId"); + if (trackId.empty()) { + result->Error("INVALID_ARGUMENT", "trackId is required"); + return; + } + if (rendererId.empty()) { + result->Error("INVALID_ARGUMENT", "rendererId is required"); + return; + } + + auto formatIt = params.find(EncodableValue("format")); + if (formatIt == params.end()) { + result->Error("INVALID_ARGUMENT", "format is required"); + return; + } + if (!TypeIs(formatIt->second)) { + result->Error("INVALID_ARGUMENT", "Failed to parse format"); + return; + } + EncodableMap formatMap = GetValue(formatIt->second); + RendererAudioFormat format = RendererAudioFormat::FromValues( + findString(formatMap, "commonFormat"), findInt(formatMap, "sampleRate"), + findInt(formatMap, "channels")); + + libwebrtc::scoped_refptr media_track = + webrtc_instance_->MediaTrackForId(trackId); + if (!media_track) { + result->Error("INVALID_ARGUMENT", "No such track"); + return; + } + + mutex_.lock(); + if (renderers_.find(rendererId) != renderers_.end()) { + mutex_.unlock(); + result->Success(flutter::EncodableValue(true)); + return; + } + + std::ostringstream oss; + oss << "io.livekit.audio.renderer/channel-" << rendererId; + + renderers_[rendererId] = std::make_unique( + messenger_, oss.str(), media_track, format); + mutex_.unlock(); + + result->Success(flutter::EncodableValue(true)); + } else if (method_call.method_name().compare("stopAudioRenderer") == 0) { + if (!method_call.arguments()) { + result->Error("Bad Arguments", "Null arguments received"); + return; + } + flutter::EncodableMap params = + GetValue(*method_call.arguments()); + std::string rendererId = findString(params, "rendererId"); + if (rendererId.empty()) { + result->Error("INVALID_ARGUMENT", "rendererId is required"); + return; + } + + mutex_.lock(); + auto it = renderers_.find(rendererId); + if (it != renderers_.end()) { + it->second->RemoveSink(); + renderers_.erase(it); + } + mutex_.unlock(); + + result->Success(flutter::EncodableValue(true)); } else { result->NotImplemented(); } diff --git a/shared_cpp/audio_renderer.cpp b/shared_cpp/audio_renderer.cpp new file mode 100644 index 000000000..3694f295c --- /dev/null +++ b/shared_cpp/audio_renderer.cpp @@ -0,0 +1,160 @@ +#include "audio_renderer.h" + +#include +#include + +namespace { + +std::vector UpsampleAudio(const int16_t *src, int src_frames, + int out_frames, int channels) { + std::vector out(static_cast(out_frames) * channels); + + // Edge case: a single source frame — just repeat it. + if (src_frames <= 1) { + for (int f = 0; f < out_frames; ++f) { + for (int ch = 0; ch < channels; ++ch) { + out[f * channels + ch] = src[ch]; + } + } + return out; + } + + double ratio = + static_cast(src_frames) / static_cast(out_frames); + + for (int f = 0; f < out_frames; ++f) { + double src_pos = f * ratio; + int idx = std::min(static_cast(src_pos), src_frames - 2); + float frac = static_cast(src_pos - idx); + + for (int ch = 0; ch < channels; ++ch) { + int16_t s0 = src[idx * channels + ch]; + int16_t s1 = src[(idx + 1) * channels + ch]; + int sample = static_cast(s0 + frac * (s1 - s0)); + sample = std::clamp(sample, static_cast(INT16_MIN), + static_cast(INT16_MAX)); + out[f * channels + ch] = static_cast(sample); + } + } + + return out; +} + +std::vector DownsampleAudio(const int16_t *src, int src_frames, + int out_frames, int src_rate, + int target_rate, int channels) { + std::vector out(static_cast(out_frames) * channels); + double ratio = + static_cast(src_rate) / static_cast(target_rate); + + for (int f = 0; f < out_frames; ++f) { + int src_start = static_cast(f * ratio); + int src_end = std::min(static_cast((f + 1) * ratio), src_frames); + + for (int ch = 0; ch < channels; ++ch) { + int64_t sum = 0; + for (int i = src_start; i < src_end; ++i) { + sum += src[i * channels + ch]; + } + int count = src_end - src_start; + out[f * channels + ch] = + count > 0 ? static_cast(std::clamp( + sum / count, INT16_MIN, INT16_MAX)) + : 0; + } + } + + return out; +} + +} // namespace + +RendererAudioFormat RendererAudioFormat::FromValues( + const std::string &common_format, int sample_rate, int channels) { + RendererAudioFormat format; + format.common_format = common_format.empty() ? "int16" : common_format; + format.sample_rate = sample_rate > 0 ? sample_rate : 48000; + format.channels = channels > 0 ? channels : 1; + return format; +} + +std::vector ResampleAudio(const int16_t *src, int src_frames, + int src_rate, int target_rate, + int channels, int &out_frames) { + if (src_rate == target_rate || src_frames <= 0 || channels <= 0) { + out_frames = src_frames; + return std::vector( + src, src + static_cast(std::max(src_frames, 0)) * channels); + } + + out_frames = static_cast((static_cast(src_frames) * + target_rate) / + src_rate); + if (out_frames <= 0) { + out_frames = 0; + return {}; + } + + if (target_rate > src_rate) { + return UpsampleAudio(src, src_frames, out_frames, channels); + } + return DownsampleAudio(src, src_frames, out_frames, src_rate, target_rate, + channels); +} + +AudioConversionResult +ConvertAudioData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + const RendererAudioFormat &target_format) { + AudioConversionResult result; + + // WebRTC AudioTrackSink always delivers 16-bit signed int16 PCM. + if (bits_per_sample != 16 || number_of_channels == 0 || + number_of_frames == 0) { + return result; + } + + int channels = static_cast(number_of_channels); + int src_frames = static_cast(number_of_frames); + const int16_t *src = reinterpret_cast(audio_data); + + int out_frames = 0; + std::vector resampled = ResampleAudio( + src, src_frames, sample_rate, target_format.sample_rate, channels, + out_frames); + if (out_frames <= 0) { + return result; + } + + int requested_channels = std::max(target_format.channels, 1); + int out_channels = std::min(requested_channels, channels); + + result.frame_length = out_frames; + result.channels = out_channels; + + if (target_format.common_format == "float32") { + result.data.resize(static_cast(out_frames) * out_channels * 4); + for (int f = 0; f < out_frames; ++f) { + for (int ch = 0; ch < out_channels; ++ch) { + float sample = resampled[f * channels + ch] / 32767.0f; + size_t offset = (static_cast(f) * out_channels + ch) * 4; + // memcpy relies on the host being little-endian, true for the + // x86/x64/ARM desktop targets this plugin builds for. + std::memcpy(&result.data[offset], &sample, sizeof(float)); + } + } + } else { + result.data.resize(static_cast(out_frames) * out_channels * 2); + for (int f = 0; f < out_frames; ++f) { + for (int ch = 0; ch < out_channels; ++ch) { + int16_t sample = resampled[f * channels + ch]; + size_t offset = (static_cast(f) * out_channels + ch) * 2; + result.data[offset] = static_cast(sample & 0xFF); + result.data[offset + 1] = static_cast((sample >> 8) & 0xFF); + } + } + } + + result.success = true; + return result; +} diff --git a/shared_cpp/audio_renderer.h b/shared_cpp/audio_renderer.h new file mode 100644 index 000000000..2b4cc7b25 --- /dev/null +++ b/shared_cpp/audio_renderer.h @@ -0,0 +1,48 @@ +#ifndef AUDIO_RENDERER_H +#define AUDIO_RENDERER_H + +#include +#include +#include +#include + +// Audio format requested by the Dart caller for a given audio renderer. +struct RendererAudioFormat { + int bits_per_sample = 16; + int sample_rate = 48000; + int channels = 1; + std::string common_format = "int16"; + + // Applies the same defaults as the Android `RendererAudioFormat.fromMap` + // factory for any missing/invalid value. + static RendererAudioFormat FromValues(const std::string &common_format, + int sample_rate, int channels); +}; + +struct AudioConversionResult { + bool success = false; + std::vector data; + int frame_length = 0; + int channels = 0; +}; + +// Resamples interleaved int16 PCM from src_rate to target_rate: linear +// interpolation when upsampling, a box filter (per-sample average) when +// downsampling. Returns the input unchanged when the rates already match. +// Sets out_frames to 0 and returns an empty vector if resampling is not +// possible (invalid frame/channel count, or the target frame count is 0). +std::vector ResampleAudio(const int16_t *src, int src_frames, + int src_rate, int target_rate, + int channels, int &out_frames); + +// Converts raw interleaved PCM audio delivered by a WebRTC AudioTrackSink +// into the renderer's target format: resample -> keep the first N channels +// -> encode as int16 or float32 little-endian bytes. `bits_per_sample` must +// be 16, since WebRTC audio sinks always deliver 16-bit signed PCM; any other +// value, or a zero channel/frame count, yields `success = false`. +AudioConversionResult +ConvertAudioData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + const RendererAudioFormat &target_format); + +#endif // AUDIO_RENDERER_H diff --git a/windows/CMakeLists.txt b/windows/CMakeLists.txt index a4590a2be..ddef2a106 100644 --- a/windows/CMakeLists.txt +++ b/windows/CMakeLists.txt @@ -28,6 +28,7 @@ add_library(${PLUGIN_NAME} SHARED "task_runner_windows.cpp" "../shared_cpp/fft_processor.cpp" "../shared_cpp/audio_visualizer.cpp" + "../shared_cpp/audio_renderer.cpp" "../shared_cpp/pffft.c" ) diff --git a/windows/livekit_plugin.cpp b/windows/livekit_plugin.cpp index ccbec94ea..c4682708d 100644 --- a/windows/livekit_plugin.cpp +++ b/windows/livekit_plugin.cpp @@ -27,10 +27,12 @@ #include #include +#include #include #include #include +#include "audio_renderer.h" #include "audio_visualizer.h" #include "task_runner_windows.h" @@ -135,6 +137,104 @@ class VisualizerSink : public libwebrtc::AudioTrackSink { int bar_count_ = 7; }; +class AudioRendererSink : public libwebrtc::AudioTrackSink { +public: + AudioRendererSink( + BinaryMessenger *messenger, std::string event_channel_name, + libwebrtc::scoped_refptr media_track, + RendererAudioFormat format) + : channel_( + std::make_unique>( + messenger, event_channel_name, + &flutter::StandardMethodCodec::GetInstance())), + media_track_(media_track), format_(format) { + task_runner_ = std::make_unique(); + auto handler = std::make_unique< + flutter::StreamHandlerFunctions>( + [&](const flutter::EncodableValue *arguments, + std::unique_ptr> + &&events) + -> std::unique_ptr< + flutter::StreamHandlerError> { + sink_ = std::move(events); + on_listen_called_ = true; + return nullptr; + }, + [&](const flutter::EncodableValue *arguments) + -> std::unique_ptr< + flutter::StreamHandlerError> { + on_listen_called_ = false; + return nullptr; + }); + + channel_->SetStreamHandler(std::move(handler)); + ((libwebrtc::RTCAudioTrack *)media_track_.get())->AddSink(this); + } + ~AudioRendererSink() override {} + +public: + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames) override { + // Unlike VisualizerSink, live audio must not be queued while no + // subscriber is attached — drop the frame instead. + if (!on_listen_called_) { + return; + } + + AudioConversionResult conversion = + ConvertAudioData(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames, format_); + if (!conversion.success) { + LogDroppedFrame(); + return; + } + + EncodableMap payload; + payload[EncodableValue("commonFormat")] = + EncodableValue(format_.common_format); + payload[EncodableValue("sampleRate")] = + EncodableValue(format_.sample_rate); + payload[EncodableValue("channels")] = EncodableValue(conversion.channels); + payload[EncodableValue("frameLength")] = + EncodableValue(conversion.frame_length); + payload[EncodableValue("data")] = EncodableValue(std::move(conversion.data)); + + PostEvent(EncodableValue(payload)); + } + + void PostEvent(const flutter::EncodableValue &event) { + std::weak_ptr> weak_sink = sink_; + task_runner_->EnqueueTask([weak_sink, event]() { + auto sink = weak_sink.lock(); + if (sink) { + sink->Success(event); + } + }); + } + + void RemoveSink() { + ((libwebrtc::RTCAudioTrack *)media_track_.get())->RemoveSink(this); + channel_->SetStreamHandler(nullptr); + } + +private: + void LogDroppedFrame() { + dropped_frame_count_ += 1; + if (dropped_frame_count_ <= 5 || dropped_frame_count_ % 100 == 0) { + std::cerr << "[LiveKit] Dropping audio frame #" << dropped_frame_count_ + << " for audio renderer" << std::endl; + } + } + + std::unique_ptr task_runner_; + std::unique_ptr> channel_; + std::shared_ptr> sink_; + bool on_listen_called_ = false; + libwebrtc::scoped_refptr media_track_; + RendererAudioFormat format_; + int64_t dropped_frame_count_ = 0; +}; + class LiveKitPlugin : public flutter::Plugin { public: static void RegisterWithRegistrar(flutter::PluginRegistrarWindows *registrar); @@ -152,6 +252,8 @@ class LiveKitPlugin : public flutter::Plugin { private: flutter_webrtc_plugin::FlutterWebRTC *webrtc_instance_ = nullptr; std::unordered_map> visualizers_; + std::unordered_map> + renderers_; BinaryMessenger *messenger_ = nullptr; mutable std::mutex mutex_; }; @@ -247,6 +349,82 @@ void LiveKitPlugin::HandleMethodCall( } result->Success(); + } else if (method_call.method_name().compare("startAudioRenderer") == 0) { + if (!method_call.arguments()) { + result->Error("Bad Arguments", "Null arguments received"); + return; + } + flutter::EncodableMap params = + GetValue(*method_call.arguments()); + std::string trackId = findString(params, "trackId"); + std::string rendererId = findString(params, "rendererId"); + if (trackId.empty()) { + result->Error("INVALID_ARGUMENT", "trackId is required"); + return; + } + if (rendererId.empty()) { + result->Error("INVALID_ARGUMENT", "rendererId is required"); + return; + } + + auto formatIt = params.find(EncodableValue("format")); + if (formatIt == params.end()) { + result->Error("INVALID_ARGUMENT", "format is required"); + return; + } + if (!TypeIs(formatIt->second)) { + result->Error("INVALID_ARGUMENT", "Failed to parse format"); + return; + } + EncodableMap formatMap = GetValue(formatIt->second); + RendererAudioFormat format = RendererAudioFormat::FromValues( + findString(formatMap, "commonFormat"), findInt(formatMap, "sampleRate"), + findInt(formatMap, "channels")); + + libwebrtc::scoped_refptr media_track = + webrtc_instance_->MediaTrackForId(trackId); + if (!media_track) { + result->Error("INVALID_ARGUMENT", "No such track"); + return; + } + + mutex_.lock(); + if (renderers_.find(rendererId) != renderers_.end()) { + mutex_.unlock(); + result->Success(flutter::EncodableValue(true)); + return; + } + + std::ostringstream oss; + oss << "io.livekit.audio.renderer/channel-" << rendererId; + + renderers_[rendererId] = std::make_unique( + messenger_, oss.str(), media_track, format); + mutex_.unlock(); + + result->Success(flutter::EncodableValue(true)); + } else if (method_call.method_name().compare("stopAudioRenderer") == 0) { + if (!method_call.arguments()) { + result->Error("Bad Arguments", "Null arguments received"); + return; + } + flutter::EncodableMap params = + GetValue(*method_call.arguments()); + std::string rendererId = findString(params, "rendererId"); + if (rendererId.empty()) { + result->Error("INVALID_ARGUMENT", "rendererId is required"); + return; + } + + mutex_.lock(); + auto it = renderers_.find(rendererId); + if (it != renderers_.end()) { + it->second->RemoveSink(); + renderers_.erase(it); + } + mutex_.unlock(); + + result->Success(flutter::EncodableValue(true)); } else { result->NotImplemented(); }