Add partial support to Audio through Miniaudio

Apps/Playground/Android/BabylonNative/src/main/java/com/babylonjs/embedding/AudioFocusHelper.java

@@ -0,0 +1,67 @@
+package com.babylonjs.embedding;
+
+import android.content.Context;
+import android.media.AudioAttributes;
+import android.media.AudioFocusRequest;
+import android.media.AudioManager;
+import android.os.Build;
+
+/**
+ * Requests Android audio focus before WebAudio playback starts.
+ */
+public final class AudioFocusHelper {
+    private static AudioManager audioManager;
+    private static AudioFocusRequest audioFocusRequest;
+    private static final AudioManager.OnAudioFocusChangeListener focusListener = focusChange -> {
+        switch (focusChange) {
+            case AudioManager.AUDIOFOCUS_LOSS:
+            case AudioManager.AUDIOFOCUS_LOSS_TRANSIENT:
+            case AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK:
+                nativeNotifyAudioInterruption(true);
+                break;
+            case AudioManager.AUDIOFOCUS_GAIN:
+                nativeNotifyAudioInterruption(false);
+                break;
+            default:
+                break;
+        }
+    };
+
+    static {
+        System.loadLibrary("BabylonNativeEmbedding");
+    }
+
+    private AudioFocusHelper() {}
+
+    public static void initialize(Context context) {
+        audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
+    }
+
+    public static void requestPlaybackFocus() {
+        if (audioManager == null) {
+            return;
+        }
+
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            if (audioFocusRequest == null) {
+                AudioAttributes attributes = new AudioAttributes.Builder()
+                        .setUsage(AudioAttributes.USAGE_GAME)
+                        .setContentType(AudioAttributes.CONTENT_TYPE_MUSIC)
+                        .build();
+                audioFocusRequest = new AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN)
+                        .setAudioAttributes(attributes)
+                        .setOnAudioFocusChangeListener(focusListener)
+                        .build();
+            }
+
+            audioManager.requestAudioFocus(audioFocusRequest);
+        } else {
+            audioManager.requestAudioFocus(
+                    focusListener,
+                    AudioManager.STREAM_MUSIC,
+                    AudioManager.AUDIOFOCUS_GAIN);
+        }
+    }
+
+    private static native void nativeNotifyAudioInterruption(boolean began);
+}

Apps/Playground/Android/app/build.gradle

@@ -82,6 +82,11 @@ tasks.named('preBuild').configure { dependsOn ':BabylonNative:assembleRelease' }
 
 tasks.register('copyFiles') {
     doLast {
+        copy {
+            from '../../../node_modules/babylonjs-addons'
+            include "babylonjs.addons.js"
+            into 'src/main/assets/Scripts'
+        }
         copy {
             from '../../../node_modules/babylonjs'
             include "babylon.max.js"

Apps/Playground/Android/app/src/main/java/com/android/babylonnative/playground/PlaygroundActivity.java

@@ -38,7 +38,8 @@ protected void onCreate(Bundle icicle) {
         // the first View attach completes engine init on the JS thread, in
         // submission order.
         loadBootstrapScripts(mRuntimeHandle);
-        BabylonNative.runtimeLoadScript(mRuntimeHandle, "app:///Scripts/experience.js");
+        BabylonNative.runtimeLoadScript(mRuntimeHandle, "app:///Scripts/audio.js");
+        BabylonNative.runtimeLoadScript(mRuntimeHandle, "app:///Scripts/playground_runner.js");
 
         mView = new BabylonView(getApplication(), mRuntimeHandle);
         setContentView(mView);

Apps/Playground/CMakeLists.txt

@@ -15,6 +15,7 @@ set(DEPENDENCIES
 
 set(SCRIPTS
     "Scripts/experience.js"
+    "Scripts/audio.js"
     "Scripts/playground_runner.js"
     "Scripts/validation_native.js"
     "Scripts/config.json")

Apps/Playground/Scripts/audio.js

@@ -0,0 +1,143 @@
+/// <reference path="../../node_modules/babylonjs/babylon.module.d.ts" />
+
+(function () {
+    if (typeof document === "undefined") {
+        globalThis.document = {
+            addEventListener: function () { },
+            removeEventListener: function () { },
+            createElement: function (tag) {
+                if (tag === "audio") {
+                    return new Audio();
+                }
+                return {};
+            },
+            createTextNode: function (text) {
+                return { nodeValue: text };
+            }
+        };
+    }
+
+    function createBeepWavArrayBuffer(sampleRate, durationSeconds, frequency, amplitude) {
+        const frameCount = Math.max(1, Math.floor(sampleRate * durationSeconds));
+        const bytesPerSample = 2;
+        const channelCount = 1;
+        const dataSize = frameCount * channelCount * bytesPerSample;
+        const buffer = new ArrayBuffer(44 + dataSize);
+        const view = new DataView(buffer);
+
+        const writeString = function (offset, value) {
+            for (let i = 0; i < value.length; i++) {
+                view.setUint8(offset + i, value.charCodeAt(i));
+            }
+        };
+
+        writeString(0, "RIFF");
+        view.setUint32(4, 36 + dataSize, true);
+        writeString(8, "WAVE");
+        writeString(12, "fmt ");
+        view.setUint32(16, 16, true);
+        view.setUint16(20, 1, true);
+        view.setUint16(22, channelCount, true);
+        view.setUint32(24, sampleRate, true);
+        view.setUint32(28, sampleRate * channelCount * bytesPerSample, true);
+        view.setUint16(32, channelCount * bytesPerSample, true);
+        view.setUint16(34, 8 * bytesPerSample, true);
+        writeString(36, "data");
+        view.setUint32(40, dataSize, true);
+
+        for (let frame = 0; frame < frameCount; frame++) {
+            const sample = Math.sin((2 * Math.PI * frequency * frame) / sampleRate) * amplitude;
+            const intSample = Math.max(-32768, Math.min(32767, Math.round(sample * 32767)));
+            view.setInt16(44 + frame * bytesPerSample, intSample, true);
+        }
+
+        return buffer;
+    }
+
+    let clickAudioContext = null;
+    let clickAudioBuffer = null;
+
+    async function playClickBeep() {
+        if (typeof AudioContext === "undefined") {
+            BABYLON.Tools.Warn("AudioContext is not available");
+            return;
+        }
+
+        if (!clickAudioContext) {
+            clickAudioContext = new AudioContext();
+        }
+
+        if (clickAudioContext.state === "suspended") {
+            await clickAudioContext.resume();
+        }
+
+        if (!clickAudioBuffer) {
+            const wavBytes = createBeepWavArrayBuffer(44100, 0.25, 880, 0.4);
+            clickAudioBuffer = await clickAudioContext.decodeAudioData(wavBytes.slice(0));
+        }
+
+        const source = clickAudioContext.createBufferSource();
+        source.buffer = clickAudioBuffer;
+        source.connect(clickAudioContext.destination);
+        source.start(clickAudioContext.currentTime);
+        BABYLON.Tools.Log("Click beep");
+    }
+
+    function createScene() {
+        const engine = new BABYLON.NativeEngine({ adaptToDeviceRatio: true });
+        const scene = new BABYLON.Scene(engine);
+        scene.createDefaultCamera(true, true, true);
+
+        const beepBuffer = createBeepWavArrayBuffer(44100, 0.25, 880, 0.4);
+        let spatialSource = null;
+        let spatialStarted = false;
+
+        scene.onPointerObservable.add(function (pointerInfo) {
+            if (pointerInfo.type !== BABYLON.PointerEventTypes.POINTERDOWN) {
+                return;
+            }
+
+            playClickBeep().catch(function (error) {
+                BABYLON.Tools.Warn("Click beep failed: " + error);
+            });
+
+            if (!spatialStarted && typeof BABYLON.Sound === "function") {
+                spatialStarted = true;
+                const legacySound = new BABYLON.Sound("beep", beepBuffer, scene, function () {
+                    BABYLON.Tools.Log("Spatial beep started after click");
+                }, {
+                    autoplay: true,
+                    loop: true,
+                    spatialSound: true,
+                    maxDistance: 20
+                });
+                spatialSource = legacySound;
+            } else if (spatialSource && typeof spatialSource.play === "function") {
+                spatialSource.play();
+            }
+        });
+
+        let angle = 0;
+        scene.onBeforeRenderObservable.add(function () {
+            if (!spatialSource) {
+                return;
+            }
+
+            angle += 0.02;
+            const x = Math.cos(angle) * 3;
+            const z = Math.sin(angle) * 3;
+
+            if (typeof spatialSource.setPosition === "function") {
+                spatialSource.setPosition(new BABYLON.Vector3(x, 0, z));
+            } else if (spatialSource._soundSource) {
+                spatialSource._soundSource.position = new BABYLON.Vector3(x, 0, z);
+            }
+        });
+
+        BABYLON.Tools.Log("Click anywhere to play a beep (and start spatial audio)");
+
+        return scene;
+    }
+
+    globalThis.createScene = createScene;
+})();

Apps/Playground/iOS/AppDelegate.swift

@@ -24,7 +24,8 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         // first BNView attach completes engine initialization on the JS
         // thread, in submission order.
         PlaygroundBootstrap.loadScripts(runtime)
-        runtime.loadScript("app:///Scripts/experience.js")
+        runtime.loadScript("app:///Scripts/audio.js")
+        runtime.loadScript("app:///Scripts/playground_runner.js")
 
         self.runtime = runtime
         return true

Apps/UnitTests/CMakeLists.txt

@@ -78,6 +78,7 @@ target_link_libraries(UnitTests
     PRIVATE NativeEncoding
     PRIVATE ScriptLoader
     PRIVATE ShaderCache
+    PRIVATE WebAudio
     PRIVATE Window
     PRIVATE XMLHttpRequest
     PRIVATE gtest_main