From 2e9eee08efa4c029be6bc3da3146adcf08fe7bc9 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:24:43 +0200 Subject: [PATCH 01/11] The first version is already working in Meet on MacOS --- .../Scripts/Audio/AudioProcessingModule.cs | 167 ++++++++++ .../Audio/AudioProcessingModule.cs.meta | 11 + Runtime/Scripts/Core/FfiFrameObserver.cs | 160 +++++++++ Runtime/Scripts/Core/FfiFrameObserver.cs.meta | 11 + Runtime/Scripts/Internal/FFI/FFIClient.cs | 4 + .../Internal/FFI/FfiRequestExtensions.cs | 13 + .../Meet/Assets/Editor/MeetManagerEditor.cs | 23 ++ Samples~/Meet/Assets/Plugins/iOS.meta | 8 + .../Assets/Plugins/iOS/AudioSessionLatency.mm | 18 ++ .../Plugins/iOS/AudioSessionLatency.mm.meta | 33 ++ Samples~/Meet/Assets/Runtime/Audio.meta | 8 + .../Runtime/Audio/AcousticEchoCanceller.cs | 220 +++++++++++++ .../Audio/AcousticEchoCanceller.cs.meta | 11 + .../Assets/Runtime/Audio/AecAudioProbe.cs | 51 +++ .../Runtime/Audio/AecAudioProbe.cs.meta | 11 + .../Assets/Runtime/Audio/AecMicrophoneHost.cs | 22 ++ .../Runtime/Audio/AecMicrophoneHost.cs.meta | 11 + .../Runtime/Audio/AecMicrophoneSource.cs | 304 ++++++++++++++++++ .../Runtime/Audio/AecMicrophoneSource.cs.meta | 11 + .../Meet/Assets/Runtime/Audio/ApmChunkPump.cs | 168 ++++++++++ .../Assets/Runtime/Audio/ApmChunkPump.cs.meta | 11 + .../Runtime/Audio/AudioProcessingDelaySeed.cs | 64 ++++ .../Audio/AudioProcessingDelaySeed.cs.meta | 11 + .../Runtime/Audio/AudioProcessingSmokeTest.cs | 61 ++++ .../Audio/AudioProcessingSmokeTest.cs.meta | 11 + .../Assets/Runtime/Audio/PcmRingBuffer.cs | 100 ++++++ .../Runtime/Audio/PcmRingBuffer.cs.meta | 11 + Samples~/Meet/Assets/Runtime/MeetManager.cs | 39 ++- Samples~/Meet/Assets/Scenes/MeetApp.unity | 6 +- 29 files changed, 1572 insertions(+), 7 deletions(-) create mode 100644 Runtime/Scripts/Audio/AudioProcessingModule.cs create mode 100644 Runtime/Scripts/Audio/AudioProcessingModule.cs.meta create mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs create mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs.meta create mode 100644 Samples~/Meet/Assets/Plugins/iOS.meta create mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm create mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta create mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs create mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs b/Runtime/Scripts/Audio/AudioProcessingModule.cs new file mode 100644 index 00000000..d83f83be --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingModule.cs @@ -0,0 +1,167 @@ +using System; +using LiveKit.Internal.FFI; +using LiveKit.Internal.FFI.Requests; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// libwebrtc's AudioProcessingModule (AEC3 echo cancellation, noise suppression, gain + /// control, high-pass filter), driven over the FFI. + /// + /// + /// Use this to run echo cancellation over a capture path that does not go through the + /// platform audio device module (e.g. Unity's Microphone): feed the audio that is + /// played out of the loudspeaker to and the captured + /// microphone audio to , which processes it in place. + /// + /// Both accept exactly one 10 ms chunk of interleaved int16 PCM ( + /// samples per channel) and nothing else. libwebrtc's own contract is a capture thread calling + /// and a render thread calling ; + /// the native module is internally synchronised for exactly that split, and the SDK's request + /// plumbing is safe to use from both. + /// + public sealed class AudioProcessingModule : IDisposable + { + /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. + public const int ChunkSizeMs = 10; + + /// + /// libwebrtc's internal processing rates. The APM resamples any other API rate onto one of + /// these itself, so this list is diagnostic information — NOT an admission requirement. See + /// . + /// + private static readonly int[] NativeSampleRates = { 8000, 16000, 32000, 48000 }; + + private readonly FfiHandle _handle; + private bool _disposed; + + /// The native handle id, for diagnostics. + public ulong Handle => (ulong)_handle.DangerousGetHandle(); + + public AudioProcessingModule( + bool echoCancellerEnabled, + bool gainControllerEnabled, + bool highPassFilterEnabled, + bool noiseSuppressionEnabled) + { + using var request = FFIBridge.Instance.NewRequest(); + var newApm = request.request; + newApm.EchoCancellerEnabled = echoCancellerEnabled; + newApm.GainControllerEnabled = gainControllerEnabled; + newApm.HighPassFilterEnabled = highPassFilterEnabled; + newApm.NoiseSuppressionEnabled = noiseSuppressionEnabled; + + using var response = request.Send(); + FfiResponse res = response; + var owned = res.NewApm?.Apm; + if (owned?.Handle == null || owned.Handle.Id == 0) + throw new InvalidOperationException("FFI returned no APM handle"); + + _handle = FfiHandle.FromOwnedHandle(owned.Handle); + } + + public static bool IsNativeSampleRate(int sampleRate) + { + foreach (var rate in NativeSampleRates) + if (rate == sampleRate) return true; + return false; + } + + /// + /// Whether the APM accepts this rate on its API surface. + /// + /// + /// The only hard requirement is that one 10 ms chunk is a whole number of samples: both + /// here and libwebrtc's own StreamConfig::num_frames() + /// derive the chunk with integer division, so a rate that is not a multiple of 100 Hz would + /// short every chunk and drift the two feeds apart. + /// + /// A non-native rate is NOT rejected — the rate goes straight into a StreamConfig + /// and libwebrtc resamples to a native processing rate internally. A 24 kHz output rate + /// (iPad) is cancelled just as well as 48 kHz. + /// + public static bool IsSupportedApiRate(int sampleRate) => + sampleRate > 0 && sampleRate % (1000 / ChunkSizeMs) == 0; + + /// Samples per channel in one APM chunk at the given rate. + public static int FrameSizeFor(int sampleRate) => sampleRate / (1000 / ChunkSizeMs); + + /// + /// Processes the near-end (capture) stream in place. is bytes, + /// not samples — the buffer is interleaved int16. Returns the FFI error, or null on success. + /// + public string ProcessStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var process = request.request; + process.ApmHandle = Handle; + process.DataPtr = (ulong)dataPtr.ToInt64(); + process.Size = (uint)byteCount; + process.SampleRate = (uint)sampleRate; + process.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessStream?.Error); + } + + /// + /// Processes the far-end (render) reference stream in place. Same buffer contract as + /// . + /// + public string ProcessReverseStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var reverse = request.request; + reverse.ApmHandle = Handle; + reverse.DataPtr = (ulong)dataPtr.ToInt64(); + reverse.Size = (uint)byteCount; + reverse.SampleRate = (uint)sampleRate; + reverse.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessReverseStream?.Error); + } + + /// + /// Seeds the render/capture delay. AEC3 runs its own correlation estimator, so this is a + /// convergence hint rather than a hard alignment. Returns the FFI error, or null on success. + /// + public string SetStreamDelayMs(int delayMs) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var delay = request.request; + delay.ApmHandle = Handle; + delay.DelayMs = delayMs; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmSetStreamDelay?.Error); + } + + private static string ErrorOrNull(string error) => string.IsNullOrEmpty(error) ? null : error; + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + GC.SuppressFinalize(this); + } + + ~AudioProcessingModule() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta new file mode 100644 index 00000000..4263c783 --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 033fc8514082a43a18e018ca3267a9b5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs b/Runtime/Scripts/Core/FfiFrameObserver.cs new file mode 100644 index 00000000..cdb46fef --- /dev/null +++ b/Runtime/Scripts/Core/FfiFrameObserver.cs @@ -0,0 +1,160 @@ +using System; +using LiveKit.Internal; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// A decoded video frame, extracted from the raw FFI event into a protobuf-free value. + /// Carries the native plane pointers and geometry only — no managed wrappers and no + /// Google.Protobuf surface, so consumers can use frames without inheriting a + /// compile-time protobuf dependency. + /// + /// + /// The DataPtr* values point at native buffers that are valid for the duration of + /// the callback ONLY — copy out + /// synchronously if you need them past return. + /// + public readonly struct RawVideoFrame + { + /// Stream handle the frame arrived on, for correlating to a . + public readonly ulong StreamHandle; + public readonly IntPtr DataPtrY, DataPtrU, DataPtrV; + public readonly int StrideY, StrideU, StrideV; + public readonly int Width, Height; + + public RawVideoFrame( + ulong streamHandle, + IntPtr dataPtrY, int strideY, + IntPtr dataPtrU, int strideU, + IntPtr dataPtrV, int strideV, + int width, int height) + { + StreamHandle = streamHandle; + DataPtrY = dataPtrY; StrideY = strideY; + DataPtrU = dataPtrU; StrideU = strideU; + DataPtrV = dataPtrV; StrideV = strideV; + Width = width; Height = height; + } + } + + /// + /// A decoded audio frame, extracted from the raw FFI event into a protobuf-free value. + /// points at interleaved S16 PCM valid for the callback duration ONLY. + /// + public readonly struct RawAudioFrame + { + /// Stream handle the frame arrived on, for correlating to an . + public readonly ulong StreamHandle; + public readonly IntPtr DataPtr; + public readonly int SamplesPerChannel, NumChannels, SampleRate; + + public RawAudioFrame(ulong streamHandle, IntPtr dataPtr, int samplesPerChannel, int numChannels, int sampleRate) + { + StreamHandle = streamHandle; + DataPtr = dataPtr; + SamplesPerChannel = samplesPerChannel; + NumChannels = numChannels; + SampleRate = sampleRate; + } + } + + /// + /// Opt-in extension point for raw decoded frames. + /// + /// + /// / are invoked + /// synchronously on the FFI callback thread from the event router, BEFORE the event's + /// FfiHandles wrap or free the underlying native buffers — so the DataPtr + /// values each frame carries are valid for the duration of the callback ONLY. + /// + /// The SDK extracts the protobuf event into the plain / + /// structs here, on the FFI thread, so subscribers consume + /// decoded frames WITHOUT a compile-time dependency on Google.Protobuf. + /// + /// Subscriber contract: + /// + /// Runs on the FFI thread, not Unity's main loop — do not touch Unity APIs. + /// Must be non-blocking; it sits in the frame-delivery hot path. + /// Must NOT retain any DataPtr past return — copy out synchronously if needed. + /// + /// + /// No subscriber == zero cost: extraction is skipped entirely when the matching delegate + /// is null. This lets consumers build native Picture-in-Picture, echo-cancellation + /// references, frame capture, custom GPU upload, or analytics on top of the decoded + /// stream without patching the SDK. + /// + public static class FfiFrameObserver + { + public static event Action VideoFrameReceived; + public static event Action AudioFrameReceived; + + internal static void Dispatch(FfiEvent ev) + { + switch (ev.MessageCase) + { + case FfiEvent.MessageOneofCase.VideoStreamEvent: + ExtractVideo(ev.VideoStreamEvent); + break; + case FfiEvent.MessageOneofCase.AudioStreamEvent: + ExtractAudio(ev.AudioStreamEvent); + break; + } + } + + private static void ExtractVideo(VideoStreamEvent vse) + { + var handler = VideoFrameReceived; + if (handler == null) return; + if (vse.MessageCase != VideoStreamEvent.MessageOneofCase.FrameReceived) return; + + var buf = vse.FrameReceived?.Buffer; + if (buf?.Info == null || buf.Info.Components.Count < 3) return; + + var info = buf.Info; + var yc = info.Components[0]; + var uc = info.Components[1]; + var vc = info.Components[2]; + if (yc.DataPtr == 0 || uc.DataPtr == 0 || vc.DataPtr == 0) return; + + Invoke(handler, new RawVideoFrame( + vse.StreamHandle, + (IntPtr)(long)yc.DataPtr, (int)yc.Stride, + (IntPtr)(long)uc.DataPtr, (int)uc.Stride, + (IntPtr)(long)vc.DataPtr, (int)vc.Stride, + (int)info.Width, (int)info.Height)); + } + + private static void ExtractAudio(AudioStreamEvent ase) + { + var handler = AudioFrameReceived; + if (handler == null) return; + if (ase.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; + + var frame = ase.FrameReceived?.Frame; + if (frame?.Info == null || frame.Info.DataPtr == 0) return; + + var info = frame.Info; + Invoke(handler, new RawAudioFrame( + ase.StreamHandle, + (IntPtr)(long)info.DataPtr, + (int)info.SamplesPerChannel, + (int)info.NumChannels, + (int)info.SampleRate)); + } + + // A subscriber exception must not escape the native callback: this runs on the FFI + // thread inside a reverse P/Invoke, where an unhandled managed exception is fatal. + private static void Invoke(Action handler, T frame) + { + try + { + handler(frame); + } + catch (Exception e) + { + Utils.Error($"FfiFrameObserver subscriber threw: {e}"); + } + } + } +} diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta b/Runtime/Scripts/Core/FfiFrameObserver.cs.meta new file mode 100644 index 00000000..e35c2080 --- /dev/null +++ b/Runtime/Scripts/Core/FfiFrameObserver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 50292b03c8d4441118cfe20a865e0e27 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFI/FFIClient.cs b/Runtime/Scripts/Internal/FFI/FFIClient.cs index 213ee1f4..ec03b01a 100644 --- a/Runtime/Scripts/Internal/FFI/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFI/FFIClient.cs @@ -375,6 +375,10 @@ internal static void RouteFfiEvent(FfiEvent response) { if (_isDisposed) return; + // Raw decoded-frame hook. Runs first, on this thread, so subscribers see the native + // buffers before any FfiHandle below wraps or frees them. No-op when unsubscribed. + FfiFrameObserver.Dispatch(response); + // Audio stream events are handled directly on the FFI callback thread // to bypass the main thread, since the audio thread consumes the data if (response.MessageCase == FfiEvent.MessageOneofCase.AudioStreamEvent) diff --git a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs index b31f4dc0..fb9ed957 100644 --- a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs +++ b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs @@ -152,6 +152,19 @@ public static void Inject(this FfiRequest ffiRequest, T request) case RemixAndResampleRequest remixAndResampleRequest: ffiRequest.RemixAndResample = remixAndResampleRequest; break; + // Audio processing module + case NewApmRequest newApmRequest: + ffiRequest.NewApm = newApmRequest; + break; + case ApmProcessStreamRequest apmProcessStreamRequest: + ffiRequest.ApmProcessStream = apmProcessStreamRequest; + break; + case ApmProcessReverseStreamRequest apmProcessReverseStreamRequest: + ffiRequest.ApmProcessReverseStream = apmProcessReverseStreamRequest; + break; + case ApmSetStreamDelayRequest apmSetStreamDelayRequest: + ffiRequest.ApmSetStreamDelay = apmSetStreamDelayRequest; + break; // PlatformAudio case NewPlatformAudioRequest newPlatformAudioRequest: ffiRequest.NewPlatformAudio = newPlatformAudioRequest; diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 1ca6c412..8211dbc2 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,6 +13,8 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; + private SerializedProperty unityEchoCancellation; + private SerializedProperty remoteAudioGain; private void OnEnable() { @@ -25,6 +27,8 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); + unityEchoCancellation = serializedObject.FindProperty("unityEchoCancellation"); + remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } public override void OnInspectorGUI() @@ -69,6 +73,25 @@ public override void OnInspectorGUI() "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); } + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Unity Audio (PlatformAudio off)", EditorStyles.boldLabel); + + // Gray out Unity audio options when PlatformAudio is enabled + using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) + { + if (platformAudioEnabled) + { + EditorGUILayout.HelpBox("Unity audio options are only used when 'Use Platform Audio' is disabled.", MessageType.Info); + } + + EditorGUILayout.PropertyField(unityEchoCancellation, new GUIContent("Echo Cancellation (AEC3)", + "Run libwebrtc's AEC3 over Unity microphone capture, using the decoded remote audio frames as the " + + "echo reference. Assumes a single remote audio stream.")); + EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", + "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + + "distort or overload the echo canceller. 0.7 is -3.1 dB.")); + } + serializedObject.ApplyModifiedProperties(); } } diff --git a/Samples~/Meet/Assets/Plugins/iOS.meta b/Samples~/Meet/Assets/Plugins/iOS.meta new file mode 100644 index 00000000..1205da93 --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 2f128f6a519a14fc0aa42bbc2d20f447 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm new file mode 100644 index 00000000..e8749f1e --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm @@ -0,0 +1,18 @@ +#import + +// Latency terms for seeding libwebrtc's AEC3 stream delay; consumed by +// Assets/Runtime/Audio/AudioProcessingDelaySeed.cs. AVAudioSession only reports meaningful +// values once the session is active; it returns 0 before that. +extern "C" { + double MeetSample_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; + } + + double MeetSample_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; + } + + double MeetSample_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; + } +} diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta new file mode 100644 index 00000000..3483a3d7 --- /dev/null +++ b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 25d054283a4734b5a817eb1b23cbea61 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + iPhone: iOS + second: + enabled: 1 + settings: + AddToEmbeddedBinaries: false + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio.meta b/Samples~/Meet/Assets/Runtime/Audio.meta new file mode 100644 index 00000000..deab7576 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 7ed59392f993648e8ae9d202fb567a84 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs new file mode 100644 index 00000000..54da05d1 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs @@ -0,0 +1,220 @@ +using System; +using System.Diagnostics; +using System.Threading; +using LiveKit; +using UnityEngine; +using Debug = UnityEngine.Debug; + +/// +/// Runs libwebrtc's AEC3 over the local microphone capture, using the decoded remote audio as +/// the echo reference. +/// +/// The loudspeaker plays the remote audio, the microphone re-captures it, and the published +/// track would carry it back to everyone. Unity's Microphone path has no echo canceller +/// anywhere (capture never goes through a platform audio device module), so cancellation is +/// done here against the two streams already available in managed code: the FFI render frames +/// (far end, via ) and OnAudioFilterRead (near end). +/// +/// Threading: runs on the Unity audio thread and +/// on the FFI callback thread. Each owns its own pump, and +/// libwebrtc's APM is built for exactly that capture/render thread split. Nothing here touches +/// a Unity API from those threads; diagnostics use , not +/// UnityEngine.Time. +/// +/// Limitation: the far-end tap is not filtered by stream, so it assumes a SINGLE remote audio +/// stream. With several remote speakers the reference becomes the interleaving of all their +/// frames and AEC3 will not converge. The remote audio must also play through Unity +/// (AudioStream) — in PlatformAudio mode no FFI audio streams exist and nothing arrives. +/// +internal sealed class AcousticEchoCanceller : IDisposable +{ + private const int DiagnosticIntervalMs = 5000; + private const int SeedIntervalMs = 2000; + private const int SeedChangeThresholdMs = 5; + + private readonly AudioProcessingModule _apm; + private readonly ApmChunkPump _capturePump; + private readonly ApmChunkPump _renderPump; + private readonly Stopwatch _clock = new Stopwatch(); + + private long _nextDiagnosticMs; + private long _nextSeedMs; + private int _seededDelayMs = -1; + private int _farEndFrames; + private int _unsupportedRateWarned; + private bool _subscribed; + private bool _disposed; + + /// Processed 10 ms capture chunks, raised on the Unity audio thread. + public event ProcessedChunkHandler CaptureProcessed; + + private AcousticEchoCanceller(AudioProcessingModule apm) + { + _apm = apm; + _capturePump = new ApmChunkPump(_apm.ProcessStream, RaiseCaptureProcessed); + _renderPump = new ApmChunkPump(_apm.ProcessReverseStream); + } + + /// + /// Creates the canceller, or returns null when the FFI cannot hand out an APM handle — the + /// caller then publishes the unprocessed microphone rather than failing to publish at all. + /// + public static AcousticEchoCanceller TryCreate() + { + try + { + var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + Debug.Log($"[AEC] APM created handle={apm.Handle}"); + return new AcousticEchoCanceller(apm); + } + catch (Exception e) + { + Debug.LogWarning($"[AEC] APM unavailable, publishing microphone unprocessed: {e.Message}"); + return null; + } + } + + public void Start() + { + if (_disposed || _subscribed) return; + + _capturePump.Reset(); + _renderPump.Reset(); + _clock.Restart(); + _nextDiagnosticMs = DiagnosticIntervalMs; + _nextSeedMs = SeedIntervalMs; + + // Forget the delay seeded before the last Stop(). A resume restarts the whole audio path, + // so the previous value describes an acoustic path that no longer exists — and carrying it + // over lets the SeedChangeThresholdMs guard in SeedStreamDelay silently skip the reseed + // below whenever the new estimate lands within 5 ms of the stale one. + _seededDelayMs = -1; + + FfiFrameObserver.AudioFrameReceived += OnFarEndFrame; + _subscribed = true; + + SeedStreamDelay(0d); + } + + public void Stop() + { + if (!_subscribed) return; + + FfiFrameObserver.AudioFrameReceived -= OnFarEndFrame; + _subscribed = false; + _clock.Reset(); + } + + /// + /// Feeds near-end capture. Unity audio thread. Returns false when the block cannot be + /// processed, and the caller must publish it unchanged — the APM needs a rate whose 10 ms + /// chunk is a whole number of samples, and device audio backends do run at odd rates. + /// + public bool TryPushCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || data == null || channels <= 0 || sampleRate <= 0) return false; + + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + WarnUnsupportedRateOnce(sampleRate); + return false; + } + + _capturePump.Push(data, channels, sampleRate); + + var captureBlockMs = data.Length / (double)channels * 1000d / sampleRate; + MaybeReseed(captureBlockMs); + MaybeLogDiagnostics(channels, sampleRate); + return true; + } + + private void WarnUnsupportedRateOnce(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + Debug.LogWarning( + $"[AEC] capture rate {sampleRate} has no whole-sample 10 ms chunk — " + + "echo cancellation disabled, publishing microphone unprocessed"); + } + + // The far-end DataPtr is valid for the duration of this callback ONLY; the pump copies out + // before doing anything else. + private void OnFarEndFrame(RawAudioFrame frame) + { + if (_disposed) return; + + Interlocked.Increment(ref _farEndFrames); + _renderPump.Push(frame.DataPtr, frame.SamplesPerChannel, frame.NumChannels, frame.SampleRate); + } + + private void RaiseCaptureProcessed(float[] data, int channels, int sampleRate) + { + CaptureProcessed?.Invoke(data, channels, sampleRate); + } + + // AVAudioSession reports zero latency until the session goes active, so the seed is + // re-evaluated on a slow cadence rather than only once at Start(). + private void MaybeReseed(double captureBlockMs) + { + if (!_clock.IsRunning) return; + + var elapsedMs = _clock.ElapsedMilliseconds; + if (elapsedMs < _nextSeedMs) return; + _nextSeedMs = elapsedMs + SeedIntervalMs; + + SeedStreamDelay(captureBlockMs); + } + + private void SeedStreamDelay(double captureBlockMs) + { + var delayMs = AudioProcessingDelaySeed.Estimate(captureBlockMs); + if (_seededDelayMs >= 0 && Math.Abs(delayMs - _seededDelayMs) < SeedChangeThresholdMs) return; + + var error = _apm.SetStreamDelayMs(delayMs); + if (error != null) + { + Debug.LogWarning($"[AEC] set_stream_delay_ms({delayMs}) failed: {error}"); + return; + } + + _seededDelayMs = delayMs; + Debug.Log($"[AEC] stream delay seeded to {delayMs}ms (captureBlock={captureBlockMs:F1}ms)"); + } + + // Reports the measured frame geometry both feeds are actually running at. Stopwatch, not + // UnityEngine.Time: this is the audio thread. + private void MaybeLogDiagnostics(int channels, int sampleRate) + { + if (!_clock.IsRunning) return; + + var elapsedMs = _clock.ElapsedMilliseconds; + if (elapsedMs < _nextDiagnosticMs) return; + _nextDiagnosticMs = elapsedMs + DiagnosticIntervalMs; + + Debug.Log( + $"[AEC] capture {channels}ch@{sampleRate} native={AudioProcessingModule.IsNativeSampleRate(sampleRate)} " + + $"chunks={_capturePump.ProcessedChunkCount} dropped={_capturePump.DroppedSamples} " + + $"failed={_capturePump.FailedChunkCount} err={_capturePump.LastError ?? "-"} | " + + $"render {_renderPump.Channels}ch@{_renderPump.SampleRate} frames={_farEndFrames} " + + $"chunks={_renderPump.ProcessedChunkCount} dropped={_renderPump.DroppedSamples} " + + $"failed={_renderPump.FailedChunkCount} err={_renderPump.LastError ?? "-"} | " + + $"delay={_seededDelayMs}ms"); + } + + public void Dispose() + { + if (_disposed) return; + + Stop(); + _disposed = true; + CaptureProcessed = null; + _capturePump.Dispose(); + _renderPump.Dispose(); + _apm.Dispose(); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta new file mode 100644 index 00000000..39aae659 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6d2e02f70cc454f188b929269a8acbc7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs new file mode 100644 index 00000000..3ae50e5b --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs @@ -0,0 +1,51 @@ +using System; +using UnityEngine; + +/// +/// Intercepts the microphone clip's audio on the Unity audio thread. +/// +/// Sample-side re-implementation of the SDK's AudioProbe, which is internal. Behaviour is +/// deliberately identical, including — without it the +/// microphone is played back through the local loudspeaker. +/// +internal sealed class AecAudioProbe : MonoBehaviour +{ + public delegate void OnAudioDelegate(float[] data, int channels, int sampleRate); + + public event OnAudioDelegate AudioRead; + + private int _sampleRate; + private volatile bool _clearAfterInvocation; + + public void ClearAfterInvocation() + { + _clearAfterInvocation = true; + } + + private void OnEnable() + { + OnAudioConfigurationChanged(false); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + } + + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + _sampleRate = AudioSettings.outputSampleRate; + } + + private void OnAudioFilterRead(float[] data, int channels) + { + AudioRead?.Invoke(data, channels, _sampleRate); + if (_clearAfterInvocation) data.AsSpan().Clear(); + } + + private void OnDestroy() + { + AudioRead = null; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta new file mode 100644 index 00000000..6c3e6728 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 21b265f6c78d54d0ebae5f213cee36a1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs new file mode 100644 index 00000000..3f4a21f4 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs @@ -0,0 +1,22 @@ +using System; +using UnityEngine; + +/// +/// Coroutine runner and application-pause relay for , attached +/// to the microphone GameObject. Stands in for the SDK's internal MonoBehaviourContext, +/// which sample code cannot reach. +/// +internal sealed class AecMicrophoneHost : MonoBehaviour +{ + public event Action Paused; + + private void OnApplicationPause(bool pause) + { + Paused?.Invoke(pause); + } + + private void OnDestroy() + { + Paused = null; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta new file mode 100644 index 00000000..0a43fb48 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4e837510d3f214ed8bc1ee291df8f7a4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs new file mode 100644 index 00000000..c9f99ef3 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections; +using LiveKit; +using UnityEngine; + +/// +/// Microphone capture source that runs AEC3 over the captured PCM before publishing it. +/// +/// The SDK's MicrophoneSource is sealed and its AudioProbe and +/// MonoBehaviourContext are internal, so the capture path is re-implemented here. Its +/// behaviours are load-bearing and preserved: clear-after-invocation (so the microphone is not +/// played back locally), the duplicate-component guard, the Microphone.GetPosition +/// readiness poll, and the pause/resume stop-restart cycle. +/// +/// The only behavioural difference is that carries APM-processed 10 ms +/// chunks instead of raw DSP blocks. When the APM is unavailable the raw blocks pass straight +/// through, so publishing never fails because of the canceller. +/// +internal sealed class AecMicrophoneSource : RtcAudioSource +{ + private readonly GameObject _sourceObject; + private readonly string _deviceName; + private readonly AecMicrophoneHost _host; + private readonly AcousticEchoCanceller _canceller; + + public override event Action AudioRead; + + private bool _disposed; + private bool _started; + + public bool EchoCancellationActive => _canceller != null; + + private AecMicrophoneSource( + string deviceName, + GameObject sourceObject, + AecMicrophoneHost host, + AcousticEchoCanceller canceller) + : base(RtcAudioSourceType.AudioSourceMicrophone) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + _host = host; + _canceller = canceller; + + if (_canceller != null) _canceller.CaptureProcessed += OnProcessedAudio; + } + + /// + /// Builds the source. The base constructor configures the native source from Unity's current + /// output configuration, which is also the format OnAudioFilterRead delivers, so the + /// published track's metadata matches the capture geometry without further alignment. + /// + /// One of . + /// GameObject that hosts the AudioSource, probe and coroutine + /// runner. Must stay alive for the source's lifetime. + public static AecMicrophoneSource Create(string deviceName, GameObject sourceObject) + { + if (sourceObject == null) throw new ArgumentNullException(nameof(sourceObject)); + + var host = sourceObject.GetComponent(); + if (host == null) host = sourceObject.AddComponent(); + + return new AecMicrophoneSource(deviceName, sourceObject, host, AcousticEchoCanceller.TryCreate()); + } + + public override void Start() + { + base.Start(); + if (_started) return; + + if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) + throw new InvalidOperationException("Microphone access not authorized"); + + _host.Paused += OnApplicationPause; + _canceller?.Start(); + RunCoroutine(StartMicrophone()); + + _started = true; + } + + public override void Stop() + { + base.Stop(); + RunCoroutine(StopMicrophone()); + if (_host != null) _host.Paused -= OnApplicationPause; + _canceller?.Stop(); + _started = false; + } + + private IEnumerator StartMicrophone() + { + if (_sourceObject == null) + { + Debug.LogError("[AEC] microphone GameObject is null, cannot start"); + yield break; + } + + if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) + { + Debug.LogError("[AEC] microphone authorization lost"); + yield break; + } + + AudioClip clip = null; + try + { + clip = Microphone.Start( + _deviceName, + loop: true, + lengthSec: 1, + frequency: SupportedCaptureFrequency()); + } + catch (Exception e) + { + Debug.LogError($"[AEC] exception starting microphone: {e.Message}"); + yield break; + } + + if (clip == null) + { + Debug.LogError("[AEC] Microphone.Start returned null, audio session may not be ready"); + yield break; + } + + // Unity's Destroy is deferred, so a resume can land here while the previous pair is still + // alive. Duplicates would double every captured block into the APM. + var existingSource = _sourceObject.GetComponent(); + if (existingSource != null) UnityEngine.Object.DestroyImmediate(existingSource); + + var existingProbe = _sourceObject.GetComponent(); + if (existingProbe != null) + { + existingProbe.AudioRead -= OnCapturedAudio; + UnityEngine.Object.DestroyImmediate(existingProbe); + } + + var source = _sourceObject.AddComponent(); + source.clip = clip; + source.loop = true; + + var probe = _sourceObject.AddComponent(); + probe.ClearAfterInvocation(); + probe.AudioRead += OnCapturedAudio; + + const float timeout = 2f; + var elapsed = 0f; + while (Microphone.GetPosition(_deviceName) <= 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.GetPosition(_deviceName) <= 0) + { + Debug.LogError($"[AEC] microphone did not start producing data after {timeout}s"); + yield break; + } + + source.Play(); + Debug.Log($"[AEC] microphone '{_deviceName}' started at {clip.frequency}Hz, echo cancellation={EchoCancellationActive}"); + } + + // The requested rate is Unity's output rate, which the device may not accept as a capture + // rate. Clamping keeps Microphone.Start working; the DSP graph resamples the clip anyway, so + // OnAudioFilterRead still delivers the output rate either way. + private int SupportedCaptureFrequency() + { + var requested = AudioSettings.outputSampleRate; + Microphone.GetDeviceCaps(_deviceName, out var min, out var max); + + // Unity reports 0/0 when the device accepts any frequency. + if (min == 0 && max == 0) return requested; + + var clamped = Mathf.Clamp(requested, min, max); + if (clamped != requested) + Debug.Log($"[AEC] capture rate clamped {requested} -> {clamped} (device caps {min}-{max})"); + + return clamped; + } + + private IEnumerator StopMicrophone() + { + if (Microphone.IsRecording(_deviceName)) + Microphone.End(_deviceName); + + if (_sourceObject != null) + { + var probe = _sourceObject.GetComponent(); + if (probe != null) + { + probe.AudioRead -= OnCapturedAudio; + UnityEngine.Object.Destroy(probe); + } + + var source = _sourceObject.GetComponent(); + if (source != null) + UnityEngine.Object.Destroy(source); + } + + Debug.Log($"[AEC] microphone '{_deviceName}' stopped"); + yield return null; + } + + // Unity audio thread. A block the canceller cannot take is published unchanged rather than + // dropped — an un-cancelled participant beats a silent one. + private void OnCapturedAudio(float[] data, int channels, int sampleRate) + { + if (_canceller != null && _canceller.TryPushCapture(data, channels, sampleRate)) return; + + AudioRead?.Invoke(data, channels, sampleRate); + } + + // Unity audio thread, via the capture pump. + private void OnProcessedAudio(float[] data, int channels, int sampleRate) + { + AudioRead?.Invoke(data, channels, sampleRate); + } + + private void OnApplicationPause(bool pause) + { + if (!_started) return; + + if (pause) + { + // Backgrounded, release the audio resources — leaving them open trips + // AVAudioSession interruption errors (FigCaptureSourceRemote -17281). + _canceller?.Stop(); + RunCoroutine(StopMicrophone()); + } + else + { + RunCoroutine(RestartMicrophone()); + } + } + + private IEnumerator RestartMicrophone() + { + yield return StopMicrophone(); + + // After a resume the iOS audio session needs time to recover from interruption. Poll for + // actual readiness instead of guessing a delay. + yield return WaitForMicrophoneReady(); + + _canceller?.Start(); + yield return StartMicrophone(); + } + + private IEnumerator WaitForMicrophoneReady() + { + const float timeout = 2f; + var elapsed = 0f; + + while (Microphone.devices.Length == 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.devices.Length == 0) + { + Debug.LogError($"[AEC] microphone devices not available after {timeout}s"); + yield break; + } + + yield return null; + } + + // The host is a component on the (caller-owned) microphone GameObject. If that object is + // already gone — scene unload, app quit — drain the coroutine synchronously so Microphone.End + // and the component cleanup still run, as the SDK's MonoBehaviourContext does. + private void RunCoroutine(IEnumerator coroutine) + { + if (_host != null) + { + _host.StartCoroutine(coroutine); + return; + } + + while (coroutine.MoveNext()) + { + if (coroutine.Current is IEnumerator nested) + RunCoroutine(nested); + } + } + + protected override void Dispose(bool disposing) + { + if (!_disposed && disposing) Stop(); + _disposed = true; + + if (_canceller != null) + { + _canceller.CaptureProcessed -= OnProcessedAudio; + _canceller.Dispose(); + } + + base.Dispose(disposing); + } + + ~AecMicrophoneSource() + { + Dispose(false); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta new file mode 100644 index 00000000..83f422c2 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 582d502ba6935413ebc5cc0683c21059 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs new file mode 100644 index 00000000..7159b8d8 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs @@ -0,0 +1,168 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit; + +/// Processes one 10 ms interleaved int16 chunk in place at . +internal delegate string ApmChunkHandler(IntPtr dataPtr, int byteCount, int sampleRate, int channels); + +/// Receives one processed 10 ms chunk as interleaved floats. +internal delegate void ProcessedChunkHandler(float[] data, int channels, int sampleRate); + +/// +/// Re-chunks a variable-size PCM feed into the fixed 10 ms frames the APM requires, processes +/// each one in place, and optionally hands the result on. +/// +/// Neither feed is 10 ms natively: capture arrives in DSP-buffer-sized blocks (1024 frames +/// ≈ 21.3 ms at 48 kHz) and render frames arrive at whatever the decoder emits. Geometry is +/// taken from the incoming buffers, never from declared constants. +/// +/// Allocation-free once the format has settled. The chunk buffer stays pinned for the pump's +/// lifetime so the APM has a stable address to process in place. +/// +internal sealed class ApmChunkPump : IDisposable +{ + private readonly ApmChunkHandler _process; + private readonly ProcessedChunkHandler _onProcessed; + private readonly int _bufferedChunks; + + private PcmRingBuffer _ring; + private short[] _chunk; + private GCHandle _chunkPin; + private IntPtr _chunkPtr; + private int _chunkSamples; + private int _chunkBytes; + + private short[] _staging; + private float[] _processed; + + private int _sampleRate; + private int _channels; + private bool _disposed; + + public int SampleRate => _sampleRate; + public int Channels => _channels; + public int ChunkSamples => _chunkSamples; + public int ProcessedChunkCount { get; private set; } + public int FailedChunkCount { get; private set; } + public int DroppedSamples => _ring?.OverflowSamples ?? 0; + public string LastError { get; private set; } + + /// Processes one chunk in place; returns an error string or null. + /// Optional consumer of the processed chunk (capture side only). + /// + /// Ring capacity in 10 ms chunks. Sets the worst-case added latency, so keep it just large + /// enough to absorb one input block plus jitter. + /// + public ApmChunkPump(ApmChunkHandler process, ProcessedChunkHandler onProcessed = null, int bufferedChunks = 8) + { + if (bufferedChunks <= 0) throw new ArgumentOutOfRangeException(nameof(bufferedChunks)); + _process = process ?? throw new ArgumentNullException(nameof(process)); + _onProcessed = onProcessed; + _bufferedChunks = bufferedChunks; + } + + /// Feeds interleaved floats (Unity capture path). + public void Push(float[] data, int channels, int sampleRate) + { + if (_disposed || data == null || data.Length == 0) return; + if (!EnsureFormat(sampleRate, channels, data.Length)) return; + + if (_staging == null || _staging.Length < data.Length) _staging = new short[data.Length]; + for (var i = 0; i < data.Length; i++) _staging[i] = FloatToS16(data[i]); + + _ring.Write(_staging, 0, data.Length); + Drain(); + } + + /// Feeds interleaved int16 straight from a native buffer (FFI render path). + public void Push(IntPtr dataPtr, int samplesPerChannel, int channels, int sampleRate) + { + if (_disposed || dataPtr == IntPtr.Zero || samplesPerChannel <= 0 || channels <= 0) return; + + var total = samplesPerChannel * channels; + if (!EnsureFormat(sampleRate, channels, total)) return; + + if (_staging == null || _staging.Length < total) _staging = new short[total]; + Marshal.Copy(dataPtr, _staging, 0, total); + + _ring.Write(_staging, 0, total); + Drain(); + } + + public void Reset() => _ring?.Clear(); + + // An FFI failure must not escape: on the capture side this runs inside OnAudioFilterRead, and + // an exception there takes out Unity's audio callback. The unprocessed chunk is forwarded + // instead, so a broken APM degrades to no cancellation rather than to no audio. + private void Drain() + { + while (_ring.TryDrain(_chunk, _chunkSamples)) + { + try + { + var error = _process(_chunkPtr, _chunkBytes, _sampleRate, _channels); + if (error != null) LastError = error; + } + catch (Exception e) + { + LastError = e.Message; + FailedChunkCount++; + } + + ProcessedChunkCount++; + + if (_onProcessed == null) continue; + + for (var i = 0; i < _chunkSamples; i++) _processed[i] = _chunk[i] / 32768f; + _onProcessed(_processed, _channels, _sampleRate); + } + } + + private bool EnsureFormat(int sampleRate, int channels, int incomingSamples) + { + if (sampleRate <= 0 || channels <= 0) return false; + if (sampleRate == _sampleRate && channels == _channels) return true; + + var frameSize = AudioProcessingModule.FrameSizeFor(sampleRate); + if (frameSize <= 0) return false; + + ReleaseChunk(); + + _sampleRate = sampleRate; + _channels = channels; + _chunkSamples = frameSize * channels; + _chunkBytes = _chunkSamples * sizeof(short); + + _chunk = new short[_chunkSamples]; + _chunkPin = GCHandle.Alloc(_chunk, GCHandleType.Pinned); + _chunkPtr = _chunkPin.AddrOfPinnedObject(); + _processed = new float[_chunkSamples]; + + // Never smaller than one input block, or a large block would immediately overflow. + var capacity = Math.Max(_chunkSamples * _bufferedChunks, incomingSamples + _chunkSamples); + _ring = new PcmRingBuffer(capacity); + return true; + } + + private void ReleaseChunk() + { + if (_chunkPin.IsAllocated) _chunkPin.Free(); + _chunkPtr = IntPtr.Zero; + _chunk = null; + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + ReleaseChunk(); + } + + private static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta new file mode 100644 index 00000000..99d2ca85 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3213055d7f9af4da4b525a2d8b9f9a1d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs new file mode 100644 index 00000000..3a4cdeb2 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs @@ -0,0 +1,64 @@ +using System; +using System.Runtime.InteropServices; + +/// +/// Computes the set_stream_delay_ms seed: +/// (t_render - t_analyze) + (t_process - t_capture). +/// +/// AEC3 runs its own correlation-based delay estimator +/// (use_external_delay_estimator = false), so this is a convergence hint, not a hard +/// alignment — but a wildly wrong value slows convergence. iOS sources the platform terms from +/// AVAudioSession (see Plugins/iOS/AudioSessionLatency.mm); Android has no +/// equivalent accessor, so it starts from the buffering this pipeline adds and lets AEC3 find +/// the rest. +/// +/// Known gap: the reference is tapped on the decoder side, so any playout queue between the +/// decoder and the loudspeaker (Unity's AudioStream ring buffer, ~30-200 ms) is not part of +/// this estimate. +/// +internal static class AudioProcessingDelaySeed +{ +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double MeetSample_AudioSessionIOBufferDuration(); +#endif + + private const int MinDelayMs = 0; + private const int MaxDelayMs = 500; + + /// Seed used when no platform latency is readable (Android, or an inactive session). + private const int FallbackDelayMs = 60; + + /// + /// Measured duration of one OnAudioFilterRead block, which is how far behind + /// real-time the capture stream already is when it reaches the APM. + /// + public static int Estimate(double captureBlockMs) + { + var platformMs = PlatformLatencyMs(); + var seed = (int)Math.Round(platformMs + captureBlockMs); + return Math.Min(MaxDelayMs, Math.Max(MinDelayMs, seed)); + } + + public static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + var output = MeetSample_AudioSessionOutputLatency(); + var input = MeetSample_AudioSessionInputLatency(); + var ioBuffer = MeetSample_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Session not yet active; fall through to the platform-agnostic seed. + } + return FallbackDelayMs; +#else + return FallbackDelayMs; +#endif + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta new file mode 100644 index 00000000..086e91bd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9636ad91efd914976b3ed12df25e7e8c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs new file mode 100644 index 00000000..859260bd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs @@ -0,0 +1,61 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit; +using UnityEngine; + +/// +/// Go/no-go check for the FFI , runnable without joining a +/// room. Exercises the full call shape: create a handle, seed a delay, and process one 10 ms +/// chunk in each direction. The APM is compiled into each platform's FFI binary separately, so +/// a per-platform surprise is far cheaper to find here than after the pipeline is wired. +/// +internal static class AudioProcessingSmokeTest +{ + public static string Run() + { + var rate = AudioSettings.outputSampleRate > 0 ? AudioSettings.outputSampleRate : 48000; + var frameSize = AudioProcessingModule.FrameSizeFor(rate); + + AudioProcessingModule apm; + try + { + apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: false, + highPassFilterEnabled: false, + noiseSuppressionEnabled: false); + } + catch (Exception e) + { + return $"FAIL create_apm: {e.GetType().Name}: {e.Message}"; + } + + using (apm) + { + var delayError = apm.SetStreamDelayMs(100); + if (delayError != null) return $"FAIL set_stream_delay_ms: {delayError}"; + + var chunk = new short[frameSize]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + + var reverseError = apm.ProcessReverseStream(ptr, bytes, rate, 1); + if (reverseError != null) return $"FAIL process_reverse_stream: {reverseError}"; + + var processError = apm.ProcessStream(ptr, bytes, rate, 1); + if (processError != null) return $"FAIL process_stream: {processError}"; + } + finally + { + pin.Free(); + } + + return $"PASS handle={apm.Handle} rate={rate} frameSize={frameSize} " + + $"nativeRate={AudioProcessingModule.IsNativeSampleRate(rate)} " + + $"delaySeed={AudioProcessingDelaySeed.Estimate(0d)}ms"; + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta new file mode 100644 index 00000000..cf3748bf --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 78f2bfe939fd54d97b797b6b28398e88 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs new file mode 100644 index 00000000..f4b3e8fd --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs @@ -0,0 +1,100 @@ +using System; + +/// +/// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. +/// +/// Allocation-free after construction — both feeds run on audio-priority threads. Sized in +/// samples (interleaved, i.e. frames × channels), not frames. +/// +/// Single producer / single consumer per direction; the near-end and far-end feeds each own +/// their own instance, so no synchronisation is needed inside. +/// +internal sealed class PcmRingBuffer +{ + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta new file mode 100644 index 00000000..8c579352 --- /dev/null +++ b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6a4b6b6ecf98e4447ab88686c0058b9a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 1dc27c71..c0bf6c0a 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -13,7 +13,9 @@ /// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// No AEC support but gives more control over audio processing. +/// Gives more control over audio processing. Optionally runs libwebrtc's AEC3 over the +/// captured audio (), using the decoded remote audio frames +/// as the echo reference; without it there is no echo cancellation in this mode. /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -44,6 +46,18 @@ public class MeetManager : MonoBehaviour [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; + [Header("Unity Audio (PlatformAudio off)")] + [Tooltip("Run libwebrtc's AEC3 (the FFI AudioProcessingModule) over Unity microphone capture, " + + "using the decoded remote audio frames as the echo reference. Assumes a single remote " + + "audio stream: with several remote speakers the reference is the interleaving of all of " + + "them and the canceller will not converge.")] + [SerializeField] private bool unityEchoCancellation = true; + [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + + "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + + "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] + [Range(0f, 1f)] + [SerializeField] private float remoteAudioGain = 0.7f; + private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; private Texture _placeholderTexture; @@ -90,6 +104,8 @@ private void Start() if (usePlatformAudio) InitializePlatformAudio(); + else if (unityEchoCancellation) + Debug.Log($"AEC smoke test: {AudioProcessingSmokeTest.Run()}"); } private void InitializePlatformAudio() @@ -372,6 +388,7 @@ private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) audioObject.transform.SetParent(_audioTrackParent); var source = audioObject.AddComponent(); + source.volume = remoteAudioGain; var audiostream = new AudioStream(audioTrack, source); _audioStreams.Add(sid, audiostream); @@ -605,15 +622,22 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log("Publishing microphone using Unity Microphone API"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC3: {unityEchoCancellation})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); - + var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // AecMicrophoneSource re-implements MicrophoneSource with an APM stage in between: + // captured blocks go through AEC3 (reference = decoded remote frames) before they reach + // the track. If the APM cannot be created it publishes the raw microphone instead. + RtcAudioSource rtcSource; + if (unityEchoCancellation) + rtcSource = AecMicrophoneSource.Create(Microphone.devices[0], audioObject); + else + rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -628,6 +652,9 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { + // Dispose before destroying the host object so the source (and, for AEC, its APM + // handle) is released now rather than by the finalizer. + rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; yield break; @@ -638,7 +665,9 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log("Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource is AecMicrophoneSource { EchoCancellationActive: true } + ? "Microphone published via Unity Microphone API (AEC3 active)" + : "Microphone published via Unity Microphone API (no AEC)"); } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 91fc1e07..3122a7c1 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -902,11 +902,13 @@ MonoBehaviour: videoTrackParent: {fileID: 2128321498} participantTilePrefab: {fileID: 4315784896331113596, guid: bec493bbc3d574c07b5bbf8dd2be26b3, type: 3} frameRate: 30 - usePlatformAudio: 1 + usePlatformAudio: 0 echoCancellation: 1 noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 + unityEchoCancellation: 1 + remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: m_ObjectHideFlags: 0 @@ -919,7 +921,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 1a1b8efb5101449b280f574f853b7459, type: 2} + _config: {fileID: 11400000, guid: 6d52b5cb4c971436098df568ef2e2c67, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 From 935c8bb84d83fa384c10dbf4183bb342e2492dd1 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:13:47 +0200 Subject: [PATCH 02/11] Code moved to SDK --- README.md | 17 + Runtime/Plugins/iOS/LiveKitAudioSession.mm | 14 + .../Scripts/Audio/AudioProcessingDelayHint.cs | 78 ++++ .../Audio/AudioProcessingDelayHint.cs.meta | 2 +- Runtime/Scripts/Audio/AudioProcessingStats.cs | 87 +++++ .../AudioProcessingStats.cs.meta} | 2 +- Runtime/Scripts/Audio/AudioProcessor.cs | 350 ++++++++++++++++++ .../Scripts/Audio/AudioProcessor.cs.meta | 2 +- Runtime/Scripts/Audio/MicrophoneSource.cs | 29 ++ Runtime/Scripts/Audio/PcmRingBuffer.cs | 118 ++++++ .../Scripts}/Audio/PcmRingBuffer.cs.meta | 2 +- Runtime/Scripts/Audio/PlatformAudioSource.cs | 20 +- Runtime/Scripts/Audio/PlayoutReference.cs | 149 ++++++++ .../Scripts/Audio/PlayoutReference.cs.meta | 2 +- Runtime/Scripts/Audio/RtcAudioSource.cs | 103 ++++-- Runtime/Scripts/Core/FfiFrameObserver.cs | 160 -------- Runtime/Scripts/Internal/FFI/FFIClient.cs | 4 - .../Meet/Assets/Editor/MeetManagerEditor.cs | 39 +- Samples~/Meet/Assets/Plugins/iOS.meta | 8 - .../Assets/Plugins/iOS/AudioSessionLatency.mm | 18 - .../Plugins/iOS/AudioSessionLatency.mm.meta | 33 -- Samples~/Meet/Assets/Runtime/Audio.meta | 8 - .../Runtime/Audio/AcousticEchoCanceller.cs | 220 ----------- .../Assets/Runtime/Audio/AecAudioProbe.cs | 51 --- .../Assets/Runtime/Audio/AecMicrophoneHost.cs | 22 -- .../Runtime/Audio/AecMicrophoneSource.cs | 304 --------------- .../Runtime/Audio/AecMicrophoneSource.cs.meta | 11 - .../Meet/Assets/Runtime/Audio/ApmChunkPump.cs | 168 --------- .../Assets/Runtime/Audio/ApmChunkPump.cs.meta | 11 - .../Runtime/Audio/AudioProcessingDelaySeed.cs | 64 ---- .../Audio/AudioProcessingDelaySeed.cs.meta | 11 - .../Runtime/Audio/AudioProcessingSmokeTest.cs | 61 --- .../Audio/AudioProcessingSmokeTest.cs.meta | 11 - .../Assets/Runtime/Audio/PcmRingBuffer.cs | 100 ----- Samples~/Meet/Assets/Runtime/MeetManager.cs | 67 ++-- Samples~/Meet/Assets/Scenes/MeetApp.unity | 3 +- Tests/EditMode/AudioProcessingTests.cs | 155 ++++++++ Tests/EditMode/AudioProcessingTests.cs.meta | 11 + Tests/EditMode/PlatformAudioTests.cs | 1 + Tests/PlayMode/AudioProcessingTests.cs | 241 ++++++++++++ Tests/PlayMode/AudioProcessingTests.cs.meta | 11 + 41 files changed, 1423 insertions(+), 1345 deletions(-) create mode 100644 Runtime/Scripts/Audio/AudioProcessingDelayHint.cs rename Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta => Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/AudioProcessingStats.cs rename Runtime/Scripts/{Core/FfiFrameObserver.cs.meta => Audio/AudioProcessingStats.cs.meta} (83%) create mode 100644 Runtime/Scripts/Audio/AudioProcessor.cs rename Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta => Runtime/Scripts/Audio/AudioProcessor.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/PcmRingBuffer.cs rename {Samples~/Meet/Assets/Runtime => Runtime/Scripts}/Audio/PcmRingBuffer.cs.meta (83%) create mode 100644 Runtime/Scripts/Audio/PlayoutReference.cs rename Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta => Runtime/Scripts/Audio/PlayoutReference.cs.meta (83%) delete mode 100644 Runtime/Scripts/Core/FfiFrameObserver.cs delete mode 100644 Samples~/Meet/Assets/Plugins/iOS.meta delete mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm delete mode 100644 Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta delete mode 100644 Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs create mode 100644 Tests/EditMode/AudioProcessingTests.cs create mode 100644 Tests/EditMode/AudioProcessingTests.cs.meta create mode 100644 Tests/PlayMode/AudioProcessingTests.cs create mode 100644 Tests/PlayMode/AudioProcessingTests.cs.meta diff --git a/README.md b/README.md index 284dee4c..ce91509a 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,23 @@ IEnumerator PublishLocalMicrophoneUnity(Room room) } ``` +#### Unity Audio Processing + +Unity's `Microphone` path does not go through a platform audio device module, so on its own it has no echo cancellation. Pass `AudioProcessingOptions` to run libwebrtc's audio processing (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) over the captured audio before it reaches the track: + +```cs +var processing = new AudioProcessingOptions +{ + EchoCancellation = true, + NoiseSuppression = true, + AutoGainControl = true, + HighPassFilter = true +}; +var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); +``` + +Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. `rtcSource.AudioProcessingStats` reports whether the reference is attached and how many chunks flowed. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. + #### Unity Audio Output ```cs diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..c18515ff 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -67,4 +67,18 @@ void LiveKit_RestoreDefaultAudioSession() { } } +/// AVAudioSession latency terms, in seconds. Used to seed the echo canceller's stream delay in +/// Unity-audio mode (see AudioProcessingDelayHint.cs). All report 0 until the session is active. +double LiveKit_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; +} + +double LiveKit_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; +} + +double LiveKit_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; +} + } diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs new file mode 100644 index 00000000..feaf914b --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; +using UnityEngine; + +namespace LiveKit +{ + /// + /// Estimates the render-to-capture delay hint handed to + /// . + /// + /// + /// AEC3 runs its own correlation-based delay estimator; the hint only sets the initial + /// alignment after a reset, so it has to be in the right ballpark rather than exact. The echo + /// path in Unity-audio mode is: tap → Unity's output queue (a + /// few DSP blocks) → device output → air → device input → Microphone clip → the + /// AudioSource reading that clip → capture probe. Only the DSP block size and, on iOS, the + /// audio session latencies are readable; the remaining terms are constants. + /// + /// Main thread only: reads . + /// + internal static class AudioProcessingDelayHint + { +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionIOBufferDuration(); +#endif + + internal const int MinDelayMs = 0; + internal const int MaxDelayMs = 500; + + /// Output queue depth assumed between the listener tap and the device, in DSP blocks. + internal const int OutputQueueBlocks = 2; + + /// + /// How far the AudioSource reading the microphone clip trails the clip's write head. + /// starts reading once Microphone.GetPosition first + /// reports data, polled at 50 ms, and that offset persists for the life of the clip. + /// + internal const int MicrophoneReadBehindMs = 50; + + /// Device input plus output latency when the platform does not report it. + internal const int FallbackDeviceLatencyMs = 30; + + public static int EstimateMs() + { + var config = AudioSettings.GetConfiguration(); + return EstimateMs(config.dspBufferSize, config.sampleRate, PlatformLatencyMs()); + } + + internal static int EstimateMs(int dspBufferSize, int sampleRate, double deviceLatencyMs) + { + var blockMs = sampleRate > 0 ? dspBufferSize * 1000.0 / sampleRate : 0.0; + var estimate = blockMs * OutputQueueBlocks + deviceLatencyMs + MicrophoneReadBehindMs; + return (int)Math.Round(Math.Min(MaxDelayMs, Math.Max(MinDelayMs, estimate))); + } + + internal static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + // AVAudioSession reports zero until the session is active, hence the fallback. + var output = LiveKit_AudioSessionOutputLatency(); + var input = LiveKit_AudioSessionInputLatency(); + var ioBuffer = LiveKit_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Fall through to the constant. + } +#endif + return FallbackDeviceLatencyMs; + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta rename to Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta index 6c3e6728..c97cd183 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 21b265f6c78d54d0ebae5f213cee36a1 +guid: b1bb08510396f4e28b12569471ea0cf6 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs b/Runtime/Scripts/Audio/AudioProcessingStats.cs new file mode 100644 index 00000000..0362a81c --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessingStats.cs @@ -0,0 +1,87 @@ +namespace LiveKit +{ + /// + /// Counters from the audio processing stage of an that was + /// created with . A snapshot; read it from any thread via + /// . + /// + public readonly struct AudioProcessingStats + { + /// + /// True while capture is actually being processed. False when the source has no processing + /// stage, is stopped, or bypasses processing because Unity's output sample rate has no + /// whole-sample 10 ms chunk (it is not a multiple of 100 Hz). + /// + public readonly bool Active; + + /// + /// True while a on an enabled AudioListener is feeding + /// the far-end reference. Without it echo cancellation has nothing to cancel; noise + /// suppression, gain control and the high-pass filter still run. + /// + public readonly bool ReferenceAttached; + + /// Format of the capture feed, as delivered by Unity's audio graph. + public readonly int CaptureSampleRate; + public readonly int CaptureChannels; + + /// Format of the playout reference feed. + public readonly int ReferenceSampleRate; + public readonly int ReferenceChannels; + + /// 10 ms capture chunks run through the module. + public readonly long CaptureChunks; + + /// 10 ms reference chunks run through the module. + public readonly long ReferenceChunks; + + /// Samples discarded because a feed outran its buffer. Non-zero means the audio thread stalled. + public readonly long DroppedCaptureSamples; + public readonly long DroppedReferenceSamples; + + /// Chunks the module rejected; those capture chunks were published unprocessed. + public readonly long FailedChunks; + + /// Most recent module error, or null. + public readonly string LastError; + + /// Render-to-capture delay hint last handed to the module, or -1 if none yet. + public readonly int StreamDelayHintMs; + + public AudioProcessingStats( + bool active, + bool referenceAttached, + int captureSampleRate, + int captureChannels, + int referenceSampleRate, + int referenceChannels, + long captureChunks, + long referenceChunks, + long droppedCaptureSamples, + long droppedReferenceSamples, + long failedChunks, + string lastError, + int streamDelayHintMs) + { + Active = active; + ReferenceAttached = referenceAttached; + CaptureSampleRate = captureSampleRate; + CaptureChannels = captureChannels; + ReferenceSampleRate = referenceSampleRate; + ReferenceChannels = referenceChannels; + CaptureChunks = captureChunks; + ReferenceChunks = referenceChunks; + DroppedCaptureSamples = droppedCaptureSamples; + DroppedReferenceSamples = droppedReferenceSamples; + FailedChunks = failedChunks; + LastError = lastError; + StreamDelayHintMs = streamDelayHintMs; + } + + public override string ToString() => + $"active={Active} reference={ReferenceAttached} " + + $"capture={CaptureChannels}ch@{CaptureSampleRate} chunks={CaptureChunks} dropped={DroppedCaptureSamples} | " + + $"reference={ReferenceChannels}ch@{ReferenceSampleRate} chunks={ReferenceChunks} dropped={DroppedReferenceSamples} | " + + $"failed={FailedChunks} lastError={LastError ?? "-"} delayHint={StreamDelayHintMs}ms"; + } +} diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta b/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta similarity index 83% rename from Runtime/Scripts/Core/FfiFrameObserver.cs.meta rename to Runtime/Scripts/Audio/AudioProcessingStats.cs.meta index e35c2080..3174751e 100644 --- a/Runtime/Scripts/Core/FfiFrameObserver.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 50292b03c8d4441118cfe20a865e0e27 +guid: 8515244a296964046ae7f54887a9aa1b MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs b/Runtime/Scripts/Audio/AudioProcessor.cs new file mode 100644 index 00000000..f9d8d34f --- /dev/null +++ b/Runtime/Scripts/Audio/AudioProcessor.cs @@ -0,0 +1,350 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Threading; +using LiveKit.Internal; +using LiveKit.Internal.FFI; +using LiveKit.Internal.Threading; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using UnityEngine; + +namespace LiveKit +{ + /// + /// The processing stage behind on an + /// : owns one , re-chunks the + /// capture and the playout reference into 10 ms frames, and hands every processed capture + /// chunk to the source for the FFI. + /// + /// + /// Threading. (from the source's audio callback) and + /// (from ) both run on the Unity + /// audio thread. Within one DSP tick the capture probes run before the listener tap, so the + /// reference for tick N arrives after the capture of tick N; that is fine because the acoustic + /// echo of tick N's playout only reaches the microphone several ticks later. , + /// and the maintenance coroutine run on the main thread and never touch the + /// ring buffers; they raise flags the audio thread acts on. Nothing here logs from the audio + /// thread — diagnostics are counters, read via . + /// + /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple + /// of 10 ms, so the chunking here is not optional, and rates whose 10 ms chunk is not a whole + /// number of samples are bypassed entirely. + /// + internal sealed class AudioProcessor : IDisposable + { + /// Receives one processed 10 ms chunk and takes ownership of the array. + internal delegate void ProcessedFrameSink(NativeArray frame, int channels, int sampleRate); + + // Ring capacity in chunks. Bounds the latency added when a DSP block is not a multiple of 10 ms. + private const int BufferedChunks = 8; + private const float MaintenanceIntervalSeconds = 2f; + + private readonly AudioProcessingModule _apm; + private readonly ProcessedFrameSink _sink; + private readonly bool _echoCancellation; + + // Guards the pinned reference chunk against Dispose racing an in-flight callback. The + // capture side needs no lock: each chunk lives in its own NativeArray owned by the sink. + private readonly object _referenceLock = new object(); + + // Capture side. Audio thread only. + private PcmRingBuffer _captureRing; + private short[] _captureStaging; + private short[] _captureChunk; + private int _captureRate; + private int _captureChannels; + private int _captureChunkSamples; + + // Reference side. Audio thread only, under _referenceLock. The chunk stays pinned so the + // module has a stable address to process in place. + private PcmRingBuffer _referenceRing; + private short[] _referenceStaging; + private short[] _referenceChunk; + private GCHandle _referenceChunkPin; + private int _referenceRate; + private int _referenceChannels; + private int _referenceChunkSamples; + + // Cross-thread flags. + private volatile bool _running; + private volatile bool _bypass; + private volatile bool _disposed; + private int _captureResetRequested; + private int _referenceResetRequested; + private int _unsupportedRateWarned; + private int _maintenanceGeneration; + + // Counters. + private long _captureChunks; + private long _referenceChunks; + private long _failedChunks; + private volatile string _lastError; + private volatile int _delayHintMs = -1; + + /// The FFI could not create the module. + public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) + { + _sink = sink ?? throw new ArgumentNullException(nameof(sink)); + _echoCancellation = options.EchoCancellation; + _apm = new AudioProcessingModule( + echoCancellerEnabled: options.EchoCancellation, + gainControllerEnabled: options.AutoGainControl, + highPassFilterEnabled: options.HighPassFilter, + noiseSuppressionEnabled: options.NoiseSuppression); + } + + /// Main thread. + public void Start() + { + if (_disposed || _running) return; + _running = true; + RequestReset(); + + if (_echoCancellation) + { + PlayoutReference.AudioRead += OnPlayoutAudio; + PlayoutReference.Acquire(); + } + + SeedDelayHint(); + MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); + } + + /// Main thread. + public void Stop() + { + if (!_running) return; + _running = false; + + if (_echoCancellation) + { + PlayoutReference.AudioRead -= OnPlayoutAudio; + PlayoutReference.Release(); + } + } + + /// + /// Any thread. Clears both feeds before their next audio callback. Call when the capture + /// path restarts (e.g. a microphone resume) so stale samples do not misalign the canceller. + /// + public void RequestReset() + { + Interlocked.Exchange(ref _captureResetRequested, 1); + Interlocked.Exchange(ref _referenceResetRequested, 1); + } + + // Periodic main-thread upkeep: re-attach the reference after scene or device changes and + // refresh the delay hint (iOS reports zero session latency until the session is active). + private IEnumerator Maintenance(int generation) + { + while (_running && !_disposed && generation == _maintenanceGeneration) + { + if (_echoCancellation) PlayoutReference.EnsureAttached(); + SeedDelayHint(); + yield return new WaitForSeconds(MaintenanceIntervalSeconds); + } + } + + private void SeedDelayHint() + { + if (!_echoCancellation || _disposed) return; + + var hint = AudioProcessingDelayHint.EstimateMs(); + if (hint == _delayHintMs) return; + + string error; + try + { + error = _apm.SetStreamDelayMs(hint); + } + catch (ObjectDisposedException) + { + return; + } + + if (error != null) + { + Utils.Warning($"AudioProcessor: set_stream_delay_ms({hint}) failed: {error}"); + return; + } + + _delayHintMs = hint; + } + + /// + /// Unity audio thread. Runs the block through the module in 10 ms chunks and forwards each + /// chunk to the sink. Returns false when the block must go out unprocessed instead: + /// processing is stopped, or bypassed for this sample rate. + /// + public bool TryProcessCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return false; + if (data == null || data.Length == 0 || channels <= 0 || sampleRate <= 0) return false; + + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + _bypass = true; + WarnUnsupportedRate(sampleRate); + return false; + } + + if (Interlocked.Exchange(ref _captureResetRequested, 0) == 1) + _captureRing?.Clear(); + + if (_captureStaging == null || sampleRate != _captureRate || channels != _captureChannels || _captureStaging.Length < data.Length) + ConfigureCapture(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _captureStaging[i] = PcmConvert.FloatToS16(data[i]); + _captureRing.Write(_captureStaging, 0, data.Length); + + while (_captureRing.TryDrain(_captureChunk, _captureChunkSamples)) + { + var frame = new NativeArray(_captureChunkSamples, Allocator.Persistent); + frame.CopyFrom(_captureChunk); + ProcessCaptureChunk(frame, sampleRate, channels); + Interlocked.Increment(ref _captureChunks); + _sink(frame, channels, sampleRate); + } + + return true; + } + + private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int channels) + { + try + { + IntPtr ptr; + unsafe + { + ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(frame); + } + var error = _apm.ProcessStream(ptr, frame.Length * sizeof(short), sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + // The chunk goes out unprocessed rather than not at all. + RecordFailure(e.Message); + } + } + + // Unity audio thread, from PlayoutReference. Must not modify data. + private void OnPlayoutAudio(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running || _bypass) return; + if (data == null || data.Length == 0 || channels <= 0) return; + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) return; + + lock (_referenceLock) + { + if (_disposed) return; + + if (Interlocked.Exchange(ref _referenceResetRequested, 0) == 1) + _referenceRing?.Clear(); + + if (_referenceStaging == null || sampleRate != _referenceRate || channels != _referenceChannels || _referenceStaging.Length < data.Length) + ConfigureReference(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _referenceStaging[i] = PcmConvert.FloatToS16(data[i]); + _referenceRing.Write(_referenceStaging, 0, data.Length); + + var byteCount = _referenceChunkSamples * sizeof(short); + while (_referenceRing.TryDrain(_referenceChunk, _referenceChunkSamples)) + { + try + { + var error = _apm.ProcessReverseStream(_referenceChunkPin.AddrOfPinnedObject(), byteCount, sampleRate, channels); + if (error != null) RecordFailure(error); + } + catch (Exception e) + { + RecordFailure(e.Message); + } + Interlocked.Increment(ref _referenceChunks); + } + } + } + + // Format changes are rare (first block, device switch); the allocations here are accepted + // on the audio thread for the same reason AudioStream sizes its buffers lazily. + private void ConfigureCapture(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _captureRate = sampleRate; + _captureChannels = channels; + _captureChunkSamples = chunkSamples; + _captureChunk = new short[chunkSamples]; + _captureStaging = new short[incomingSamples]; + // Never smaller than one input block, or a large block would overflow immediately. + _captureRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void ConfigureReference(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _referenceRate = sampleRate; + _referenceChannels = channels; + _referenceChunkSamples = chunkSamples; + + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = new short[chunkSamples]; + _referenceChunkPin = GCHandle.Alloc(_referenceChunk, GCHandleType.Pinned); + + _referenceStaging = new short[incomingSamples]; + _referenceRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void RecordFailure(string error) + { + _lastError = error; + Interlocked.Increment(ref _failedChunks); + } + + // Logging is not allowed on the audio thread; hand the message to the main thread. + private void WarnUnsupportedRate(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + var message = $"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + + "audio processing is bypassed and the capture is published unprocessed."; + var context = FfiClient.Instance._context; + if (context != null) + context.Post(static m => Utils.Warning(m), message); + else + Utils.Warning(message); + } + + public AudioProcessingStats GetStats() => new AudioProcessingStats( + active: _running && !_bypass && !_disposed, + referenceAttached: _echoCancellation && PlayoutReference.IsAttached, + captureSampleRate: _captureRate, + captureChannels: _captureChannels, + referenceSampleRate: _referenceRate, + referenceChannels: _referenceChannels, + captureChunks: Interlocked.Read(ref _captureChunks), + referenceChunks: Interlocked.Read(ref _referenceChunks), + droppedCaptureSamples: _captureRing?.OverflowSamples ?? 0, + droppedReferenceSamples: _referenceRing?.OverflowSamples ?? 0, + failedChunks: Interlocked.Read(ref _failedChunks), + lastError: _lastError, + streamDelayHintMs: _delayHintMs); + + public void Dispose() + { + if (_disposed) return; + + Stop(); + lock (_referenceLock) + { + _disposed = true; + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = null; + _referenceRing = null; + } + _apm.Dispose(); + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta b/Runtime/Scripts/Audio/AudioProcessor.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta rename to Runtime/Scripts/Audio/AudioProcessor.cs.meta index 0a43fb48..593e25f5 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs.meta +++ b/Runtime/Scripts/Audio/AudioProcessor.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 4e837510d3f214ed8bc1ee291df8f7a4 +guid: 0154e5d4b40264653a6b7832d5640b7a MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 75bec1f0..9f88ad38 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -11,6 +11,13 @@ namespace LiveKit /// /// /// Ensure microphone permissions are granted before calling . + /// + /// Unity's Microphone path does not go through a platform audio device module, so on + /// its own it has no echo cancellation. Construct the source with + /// to run libwebrtc's audio processing over the capture; + /// echo cancellation then uses the mix Unity plays as its reference (see + /// ), which covers every remote and the + /// application's own audio. /// sealed public class MicrophoneSource : RtcAudioSource { @@ -35,6 +42,26 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu _sourceObject = sourceObject; } + /// + /// Creates a microphone source whose capture is run through libwebrtc's audio processing + /// (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) before it + /// reaches the track. + /// + /// The name of the device to capture from. Use to + /// get the list of available devices. + /// The GameObject to attach the AudioSource to. The object must be kept in the scene + /// for the duration of the source's lifetime. + /// Which stages to enable. With + /// the SDK attaches a to the active to obtain the + /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is + /// bypassed with a warning. See for diagnostics. + public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing) + : base(RtcAudioSourceType.AudioSourceMicrophone, processing) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + } + /// /// Begins capturing audio from the microphone. /// @@ -207,6 +234,8 @@ private IEnumerator RestartMicrophone() // recover from interruption. Poll for readiness instead of using arbitrary delay. yield return WaitForMicrophoneReady(); + // A resume is a new audio path: drop whatever the processing stage buffered before. + ResetAudioProcessing(); yield return StartMicrophone(); } diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs b/Runtime/Scripts/Audio/PcmRingBuffer.cs new file mode 100644 index 00000000..a9acddbb --- /dev/null +++ b/Runtime/Scripts/Audio/PcmRingBuffer.cs @@ -0,0 +1,118 @@ +using System; + +namespace LiveKit +{ + /// + /// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. Re-chunks Unity's + /// DSP-block-sized audio into the 10 ms frames the requires. + /// + /// + /// Allocation-free after construction: both users run on the Unity audio thread. Sized in + /// samples (frames × channels), not frames. Single producer and single consumer per instance; + /// the capture and reference feeds each own one, so there is no synchronisation inside. + /// + internal sealed class PcmRingBuffer + { + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } + } + + /// Sample format conversions shared by the capture paths. + internal static class PcmConvert + { + /// Float [-1, 1] to int16 with clamping and round-half-away-from-zero. + public static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } + + public static float S16ToFloat(short v) => v / 32768f; + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta rename to Runtime/Scripts/Audio/PcmRingBuffer.cs.meta index 8c579352..f096bd2b 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs.meta +++ b/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6a4b6b6ecf98e4447ab88686c0058b9a +guid: cce610429285242e193622fd7e37bb80 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 0b507c4c..5a7066fc 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -7,7 +7,10 @@ namespace LiveKit { /// - /// Options for audio processing when creating a PlatformAudioSource. + /// Options for libwebrtc's audio processing. Used by , where + /// the ADM applies them, and by Unity-audio sources such as + /// created with options, where the SDK runs the over the + /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions { @@ -17,7 +20,16 @@ public struct AudioProcessingOptions public bool NoiseSuppression; /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; - /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true. + /// + /// Enable the high-pass filter, which removes DC offset and low-frequency rumble ahead of + /// the other stages. Unity-audio sources only; ignores it. + /// Default: true. + /// + public bool HighPassFilter; + /// + /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + /// only. Default: true. + /// public bool PreferHardware; /// @@ -28,8 +40,12 @@ public struct AudioProcessingOptions EchoCancellation = true, NoiseSuppression = true, AutoGainControl = true, + HighPassFilter = true, PreferHardware = true }; + + /// Whether any stage of the Unity-audio processing pipeline is enabled. + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl || HighPassFilter; } /// diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs b/Runtime/Scripts/Audio/PlayoutReference.cs new file mode 100644 index 00000000..2693b4a7 --- /dev/null +++ b/Runtime/Scripts/Audio/PlayoutReference.cs @@ -0,0 +1,149 @@ +using System.Collections; +using UnityEngine; +using LiveKit.Internal; +using LiveKit.Internal.Threading; + +namespace LiveKit +{ + /// + /// Taps the final mix Unity sends to the audio device and feeds it to the echo canceller as + /// the far-end reference. Lives on the GameObject of the active . + /// + /// + /// An created with + /// attaches this component to the active listener when it starts and re-attaches it after + /// scene loads and audio device changes. Adding it to the listener yourself is supported and + /// does the same thing. + /// + /// Because the tap sits after every AudioSource, mixer group and spatializer, the reference is + /// exactly what the loudspeaker plays: every remote participant plus the game's own audio. + /// The capture probe clears its buffer after reading, so the + /// local microphone never appears in the mix. + /// + /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so + /// the sample rate and listener state are cached on the main thread. + /// + [AddComponentMenu("LiveKit/Playout Reference")] + public sealed class PlayoutReference : MonoBehaviour + { + internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); + + /// + /// Raised on the Unity audio thread with the final mix. Subscribers must not modify the + /// buffer: it is on its way to the speaker. + /// + internal static event PlayoutAudioDelegate AudioRead; + + private static PlayoutReference _active; + private static int _consumers; + + private AudioListener _listener; + private volatile int _sampleRate; + private volatile bool _deliver; + + /// Whether a reference on an enabled listener is delivering audio. + internal static bool IsAttached => _active != null && _active._deliver; + + /// Main thread. Registers a consumer and attaches to the listener if possible. + internal static void Acquire() + { + _consumers++; + EnsureAttached(); + } + + /// Main thread. The component stays on the listener; it is inert without consumers. + internal static void Release() + { + if (_consumers > 0) _consumers--; + } + + /// + /// Main thread. Attaches to the active AudioListener unless a working reference already + /// exists. No-op without consumers or without a listener; consumers call this periodically, + /// which is what covers scene loads and a destroyed listener. + /// + internal static void EnsureAttached() + { + if (_consumers == 0) return; + if (_active != null && _active.isActiveAndEnabled && + _active._listener != null && _active._listener.isActiveAndEnabled) + return; + + var listener = FindActiveListener(); + if (listener == null) return; + + var existing = listener.GetComponent(); + _active = existing != null ? existing : listener.gameObject.AddComponent(); + } + + private static AudioListener FindActiveListener() + { + var listeners = FindObjectsByType(FindObjectsSortMode.None); + foreach (var listener in listeners) + { + if (listener.isActiveAndEnabled) return listener; + } + return null; + } + + private void OnEnable() + { + _listener = GetComponent(); + if (_listener == null) + Utils.Warning("PlayoutReference must be on the AudioListener's GameObject; it will not deliver a reference from here."); + + RefreshDeliveryState(); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + if (_active == null) _active = this; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + _deliver = false; + if (_active == this) _active = null; + } + + private void Update() + { + RefreshDeliveryState(); + } + + // Listener state and the output rate are Unity APIs; sample them here for the audio thread. + private void RefreshDeliveryState() + { + _sampleRate = AudioSettings.outputSampleRate; + _deliver = _listener != null && _listener.isActiveAndEnabled; + } + + // Unity rebuilds the DSP graph on a device change (or AudioSettings.Reset), which can leave + // filter nodes detached; AudioStream recreates its probe for the same reason. Recreate this + // component so the tap is registered on the new graph. Only done while something consumes + // the reference, so a hand-placed component in an idle scene is left alone. + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + RefreshDeliveryState(); + if (_consumers == 0) return; + + var host = gameObject; + Destroy(this); + MonoBehaviourContext.RunCoroutine(Reattach(host)); + } + + private static IEnumerator Reattach(GameObject host) + { + // Let the deferred Destroy apply before adding the replacement. + yield return null; + if (host == null || _consumers == 0) yield break; + if (host.GetComponent() == null) + _active = host.AddComponent(); + } + + // Unity audio thread. + private void OnAudioFilterRead(float[] data, int channels) + { + if (!_deliver) return; + AudioRead?.Invoke(data, channels, _sampleRate); + } + } +} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta b/Runtime/Scripts/Audio/PlayoutReference.cs.meta similarity index 83% rename from Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta rename to Runtime/Scripts/Audio/PlayoutReference.cs.meta index 39aae659..a74f1070 100644 --- a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs.meta +++ b/Runtime/Scripts/Audio/PlayoutReference.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6d2e02f70cc454f188b929269a8acbc7 +guid: 44cbde2518fe54f5ebd5f24f65d8a070 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 9147b431..ee764916 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -49,12 +49,23 @@ private sealed class PendingAudioFrame private readonly RtcAudioSourceType _sourceType; public RtcAudioSourceType SourceType => _sourceType; + + /// + /// Whether this source runs libwebrtc's audio processing over its capture. False when it was + /// created without , or when the module could not be + /// created and the source fell back to unprocessed capture. + /// + public bool AudioProcessingEnabled => _processor != null; + + /// Counters from the audio processing stage; default when processing is off. + public AudioProcessingStats AudioProcessingStats => _processor?.GetStats() ?? default; private readonly int _debugId = Interlocked.Increment(ref nextDebugId); internal readonly uint _expectedSampleRate; internal readonly uint _expectedChannels; internal readonly FfiHandle Handle; protected AudioSourceInfo _info; + private readonly AudioProcessor _processor; // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM // pointer after request.Send() returns and encode it later on another queue. Because of @@ -72,16 +83,29 @@ private sealed class PendingAudioFrame private bool _started = false; private volatile bool _disposed = false; private int _audioReadCount = 0; + private int _sentFrameCount = 0; // Device-capture sources (microphone, AudioSource taps) don't know their format ahead of // time — it is whatever Unity's audio graph delivers. They use this constructor, which // configures the native source from Unity's current output configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType) - : this(audioSourceType, 0, 0) { } + : this(audioSourceType, 0, 0, null) { } + + /// + /// Device-capture source whose audio is run through libwebrtc's audio processing (echo + /// cancellation, noise suppression, gain control, high-pass filter) before it reaches the + /// track. See . If the module cannot be created the + /// source logs a warning and captures unprocessed. + /// + protected RtcAudioSource(RtcAudioSourceType audioSourceType, AudioProcessingOptions processing) + : this(audioSourceType, 0, 0, processing) { } // Sources that generate a fixed, known format (e.g. test signal generators) declare it // directly. Passing 0 for either value falls back to the device configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels) + : this(audioSourceType, sampleRate, channels, null) { } + + protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels, AudioProcessingOptions? processing) { _sourceType = audioSourceType; @@ -110,6 +134,19 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui _info = res.NewAudioSource.Source.Info; Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); Utils.Debug($"{DebugTag} created handle={Handle.DangerousGetHandle()} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + + if (processing is { } options && options.AnyProcessingEnabled) + { + try + { + _processor = new AudioProcessor(options, SendProcessedFrame); + } + catch (Exception e) + { + // Publish unprocessed rather than not at all. + Utils.Warning($"{DebugTag} audio processing unavailable, capturing unprocessed: {e.Message}"); + } + } } // Reads Unity's actual output audio configuration. The capture path delivers buffers at the @@ -150,6 +187,7 @@ public virtual void Start() { if (_started) return; AudioRead += OnAudioRead; + _processor?.Start(); _started = true; Utils.Debug($"{DebugTag} start"); } @@ -161,6 +199,7 @@ public virtual void Stop() { if (!_started) return; AudioRead -= OnAudioRead; + _processor?.Stop(); _started = false; var pendingCount = PendingFrameCount(); if (pendingCount > 0) @@ -174,45 +213,59 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_muted) return; if (_disposed) return; - var frameIndex = Interlocked.Increment(ref _audioReadCount); + var readIndex = Interlocked.Increment(ref _audioReadCount); if (channels <= 0) { - Utils.Warning($"{DebugTag} dropping audio frame #{frameIndex} because channels={channels}"); + Utils.Warning($"{DebugTag} dropping audio frame #{readIndex} because channels={channels}"); return; } if (data.Length == 0 || data.Length % channels != 0) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} has invalid shape samples={data.Length} channels={channels}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} has invalid shape samples={data.Length} channels={channels}"); return; } if ((uint)sampleRate != _expectedSampleRate || (uint)channels != _expectedChannels) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); } - var pendingBeforeSend = PendingFrameCount(); - if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) - { - Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={data.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); - } + // Optional processing stage: the block is re-chunked into 10 ms frames, run through the + // module and delivered to SendFrame one chunk at a time via SendProcessedFrame. + if (_processor != null && _processor.TryProcessCapture(data, channels, sampleRate)) + return; // Each captured frame gets its own backing buffer so the native encoder can safely // consume it asynchronously after request.Send() returns. var frameData = new NativeArray(data.Length, Allocator.Persistent); + for (int i = 0; i < data.Length; i++) + frameData[i] = PcmConvert.FloatToS16(data[i]); + + SendFrame(frameData, channels, sampleRate); + } - // Copy from the audio read buffer into the frame buffer, converting - // each sample to a 16-bit signed integer. - static short FloatToS16(float v) + // Audio thread, from the processing stage. Owns the frame from here on. + private void SendProcessedFrame(NativeArray frame, int channels, int sampleRate) + { + if (_disposed || _muted) { - v *= 32768f; - v = Math.Min(v, 32767f); - v = Math.Max(v, -32768f); - return (short)(v + Math.Sign(v) * 0.5f); + frame.Dispose(); + return; + } + SendFrame(frame, channels, sampleRate); + } + + // Hands one int16 frame to the native source. Takes ownership of frameData: it is released + // when the CaptureAudioFrame callback completes, is canceled, or the send fails. + private void SendFrame(NativeArray frameData, int channels, int sampleRate) + { + var frameIndex = Interlocked.Increment(ref _sentFrameCount); + var pendingBeforeSend = PendingFrameCount(); + if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) + { + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frameData.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); } - for (int i = 0; i < data.Length; i++) - frameData[i] = FloatToS16(data[i]); // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); @@ -228,7 +281,7 @@ static short FloatToS16(float v) } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; - pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; + pushFrame.Buffer.SamplesPerChannel = (uint)frameData.Length / (uint)channels; // Wait for async callback, log an error if the capture fails. The callback's AsyncId // echoes the RequestAsyncId that Unity wrote onto the request. @@ -239,7 +292,7 @@ static short FloatToS16(float v) FrameIndex = frameIndex, SampleRate = sampleRate, Channels = channels, - SampleCount = data.Length, + SampleCount = frameData.Length, StartedTimestamp = Stopwatch.GetTimestamp(), }; lock (_pendingFrameDataLock) @@ -292,6 +345,13 @@ void OnCanceled() } } + /// + /// Clears the audio processing stage's buffers. Call after the capture path restarts (e.g. a + /// microphone resume) so stale samples do not misalign the echo canceller. No-op without + /// processing. + /// + protected void ResetAudioProcessing() => _processor?.RequestReset(); + /// /// Mutes or unmutes the audio source. /// @@ -328,6 +388,7 @@ protected virtual void Dispose(bool disposing) } _pendingFrameData.Clear(); } + _processor?.Dispose(); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); diff --git a/Runtime/Scripts/Core/FfiFrameObserver.cs b/Runtime/Scripts/Core/FfiFrameObserver.cs deleted file mode 100644 index cdb46fef..00000000 --- a/Runtime/Scripts/Core/FfiFrameObserver.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using LiveKit.Internal; -using LiveKit.Proto; - -namespace LiveKit -{ - /// - /// A decoded video frame, extracted from the raw FFI event into a protobuf-free value. - /// Carries the native plane pointers and geometry only — no managed wrappers and no - /// Google.Protobuf surface, so consumers can use frames without inheriting a - /// compile-time protobuf dependency. - /// - /// - /// The DataPtr* values point at native buffers that are valid for the duration of - /// the callback ONLY — copy out - /// synchronously if you need them past return. - /// - public readonly struct RawVideoFrame - { - /// Stream handle the frame arrived on, for correlating to a . - public readonly ulong StreamHandle; - public readonly IntPtr DataPtrY, DataPtrU, DataPtrV; - public readonly int StrideY, StrideU, StrideV; - public readonly int Width, Height; - - public RawVideoFrame( - ulong streamHandle, - IntPtr dataPtrY, int strideY, - IntPtr dataPtrU, int strideU, - IntPtr dataPtrV, int strideV, - int width, int height) - { - StreamHandle = streamHandle; - DataPtrY = dataPtrY; StrideY = strideY; - DataPtrU = dataPtrU; StrideU = strideU; - DataPtrV = dataPtrV; StrideV = strideV; - Width = width; Height = height; - } - } - - /// - /// A decoded audio frame, extracted from the raw FFI event into a protobuf-free value. - /// points at interleaved S16 PCM valid for the callback duration ONLY. - /// - public readonly struct RawAudioFrame - { - /// Stream handle the frame arrived on, for correlating to an . - public readonly ulong StreamHandle; - public readonly IntPtr DataPtr; - public readonly int SamplesPerChannel, NumChannels, SampleRate; - - public RawAudioFrame(ulong streamHandle, IntPtr dataPtr, int samplesPerChannel, int numChannels, int sampleRate) - { - StreamHandle = streamHandle; - DataPtr = dataPtr; - SamplesPerChannel = samplesPerChannel; - NumChannels = numChannels; - SampleRate = sampleRate; - } - } - - /// - /// Opt-in extension point for raw decoded frames. - /// - /// - /// / are invoked - /// synchronously on the FFI callback thread from the event router, BEFORE the event's - /// FfiHandles wrap or free the underlying native buffers — so the DataPtr - /// values each frame carries are valid for the duration of the callback ONLY. - /// - /// The SDK extracts the protobuf event into the plain / - /// structs here, on the FFI thread, so subscribers consume - /// decoded frames WITHOUT a compile-time dependency on Google.Protobuf. - /// - /// Subscriber contract: - /// - /// Runs on the FFI thread, not Unity's main loop — do not touch Unity APIs. - /// Must be non-blocking; it sits in the frame-delivery hot path. - /// Must NOT retain any DataPtr past return — copy out synchronously if needed. - /// - /// - /// No subscriber == zero cost: extraction is skipped entirely when the matching delegate - /// is null. This lets consumers build native Picture-in-Picture, echo-cancellation - /// references, frame capture, custom GPU upload, or analytics on top of the decoded - /// stream without patching the SDK. - /// - public static class FfiFrameObserver - { - public static event Action VideoFrameReceived; - public static event Action AudioFrameReceived; - - internal static void Dispatch(FfiEvent ev) - { - switch (ev.MessageCase) - { - case FfiEvent.MessageOneofCase.VideoStreamEvent: - ExtractVideo(ev.VideoStreamEvent); - break; - case FfiEvent.MessageOneofCase.AudioStreamEvent: - ExtractAudio(ev.AudioStreamEvent); - break; - } - } - - private static void ExtractVideo(VideoStreamEvent vse) - { - var handler = VideoFrameReceived; - if (handler == null) return; - if (vse.MessageCase != VideoStreamEvent.MessageOneofCase.FrameReceived) return; - - var buf = vse.FrameReceived?.Buffer; - if (buf?.Info == null || buf.Info.Components.Count < 3) return; - - var info = buf.Info; - var yc = info.Components[0]; - var uc = info.Components[1]; - var vc = info.Components[2]; - if (yc.DataPtr == 0 || uc.DataPtr == 0 || vc.DataPtr == 0) return; - - Invoke(handler, new RawVideoFrame( - vse.StreamHandle, - (IntPtr)(long)yc.DataPtr, (int)yc.Stride, - (IntPtr)(long)uc.DataPtr, (int)uc.Stride, - (IntPtr)(long)vc.DataPtr, (int)vc.Stride, - (int)info.Width, (int)info.Height)); - } - - private static void ExtractAudio(AudioStreamEvent ase) - { - var handler = AudioFrameReceived; - if (handler == null) return; - if (ase.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - - var frame = ase.FrameReceived?.Frame; - if (frame?.Info == null || frame.Info.DataPtr == 0) return; - - var info = frame.Info; - Invoke(handler, new RawAudioFrame( - ase.StreamHandle, - (IntPtr)(long)info.DataPtr, - (int)info.SamplesPerChannel, - (int)info.NumChannels, - (int)info.SampleRate)); - } - - // A subscriber exception must not escape the native callback: this runs on the FFI - // thread inside a reverse P/Invoke, where an unhandled managed exception is fatal. - private static void Invoke(Action handler, T frame) - { - try - { - handler(frame); - } - catch (Exception e) - { - Utils.Error($"FfiFrameObserver subscriber threw: {e}"); - } - } - } -} diff --git a/Runtime/Scripts/Internal/FFI/FFIClient.cs b/Runtime/Scripts/Internal/FFI/FFIClient.cs index ec03b01a..213ee1f4 100644 --- a/Runtime/Scripts/Internal/FFI/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFI/FFIClient.cs @@ -375,10 +375,6 @@ internal static void RouteFfiEvent(FfiEvent response) { if (_isDisposed) return; - // Raw decoded-frame hook. Runs first, on this thread, so subscribers see the native - // buffers before any FfiHandle below wraps or frees them. No-op when unsubscribed. - FfiFrameObserver.Dispatch(response); - // Audio stream events are handled directly on the FFI callback thread // to bypass the main thread, since the audio thread consumes the data if (response.MessageCase == FfiEvent.MessageOneofCase.AudioStreamEvent) diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 8211dbc2..0fb70078 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,7 +13,6 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; - private SerializedProperty unityEchoCancellation; private SerializedProperty remoteAudioGain; private void OnEnable() @@ -27,7 +26,6 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); - unityEchoCancellation = serializedObject.FindProperty("unityEchoCancellation"); remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } @@ -51,26 +49,24 @@ public override void OnInspectorGUI() "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")); EditorGUILayout.Space(); - EditorGUILayout.LabelField("Audio Processing (PlatformAudio only)", EditorStyles.boldLabel); + EditorGUILayout.LabelField("Audio Processing", EditorStyles.boldLabel); - // Gray out audio processing options when PlatformAudio is disabled bool platformAudioEnabled = usePlatformAudio.boolValue; + EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", + "Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")); + EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", + "Enable noise suppression to remove background noise.")); + EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", + "Enable auto gain control to normalize audio levels.")); + + // Hardware processing is an ADM feature; gray it out when PlatformAudio is disabled. using (new EditorGUI.DisabledGroupScope(!platformAudioEnabled)) { - if (!platformAudioEnabled) - { - EditorGUILayout.HelpBox("Audio processing options are only available when 'Use Platform Audio' is enabled.", MessageType.Info); - } - - EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", - "Enable echo cancellation to remove echo from speaker playback.")); - EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", - "Enable noise suppression to remove background noise.")); - EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", - "Enable auto gain control to normalize audio levels.")); EditorGUILayout.PropertyField(preferHardwareProcessing, new GUIContent("Prefer Hardware Processing", - "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); + "PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have " + + "different quality characteristics.")); } EditorGUILayout.Space(); @@ -79,14 +75,11 @@ public override void OnInspectorGUI() // Gray out Unity audio options when PlatformAudio is enabled using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) { - if (platformAudioEnabled) - { - EditorGUILayout.HelpBox("Unity audio options are only used when 'Use Platform Audio' is disabled.", MessageType.Info); - } + EditorGUILayout.HelpBox(platformAudioEnabled + ? "Unity audio options are only used when 'Use Platform Audio' is disabled." + : "Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is the mix on the " + + "AudioListener (a PlayoutReference component is attached automatically).", MessageType.Info); - EditorGUILayout.PropertyField(unityEchoCancellation, new GUIContent("Echo Cancellation (AEC3)", - "Run libwebrtc's AEC3 over Unity microphone capture, using the decoded remote audio frames as the " + - "echo reference. Assumes a single remote audio stream.")); EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + "distort or overload the echo canceller. 0.7 is -3.1 dB.")); diff --git a/Samples~/Meet/Assets/Plugins/iOS.meta b/Samples~/Meet/Assets/Plugins/iOS.meta deleted file mode 100644 index 1205da93..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 2f128f6a519a14fc0aa42bbc2d20f447 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm deleted file mode 100644 index e8749f1e..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm +++ /dev/null @@ -1,18 +0,0 @@ -#import - -// Latency terms for seeding libwebrtc's AEC3 stream delay; consumed by -// Assets/Runtime/Audio/AudioProcessingDelaySeed.cs. AVAudioSession only reports meaningful -// values once the session is active; it returns 0 before that. -extern "C" { - double MeetSample_AudioSessionOutputLatency() { - return [[AVAudioSession sharedInstance] outputLatency]; - } - - double MeetSample_AudioSessionInputLatency() { - return [[AVAudioSession sharedInstance] inputLatency]; - } - - double MeetSample_AudioSessionIOBufferDuration() { - return [[AVAudioSession sharedInstance] IOBufferDuration]; - } -} diff --git a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta b/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta deleted file mode 100644 index 3483a3d7..00000000 --- a/Samples~/Meet/Assets/Plugins/iOS/AudioSessionLatency.mm.meta +++ /dev/null @@ -1,33 +0,0 @@ -fileFormatVersion: 2 -guid: 25d054283a4734b5a817eb1b23cbea61 -PluginImporter: - externalObjects: {} - serializedVersion: 2 - iconMap: {} - executionOrder: {} - defineConstraints: [] - isPreloaded: 0 - isOverridable: 0 - isExplicitlyReferenced: 0 - validateReferences: 1 - platformData: - - first: - Any: - second: - enabled: 0 - settings: {} - - first: - Editor: Editor - second: - enabled: 0 - settings: - DefaultValueInitialized: true - - first: - iPhone: iOS - second: - enabled: 1 - settings: - AddToEmbeddedBinaries: false - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio.meta b/Samples~/Meet/Assets/Runtime/Audio.meta deleted file mode 100644 index deab7576..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 7ed59392f993648e8ae9d202fb567a84 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs b/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs deleted file mode 100644 index 54da05d1..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AcousticEchoCanceller.cs +++ /dev/null @@ -1,220 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading; -using LiveKit; -using UnityEngine; -using Debug = UnityEngine.Debug; - -/// -/// Runs libwebrtc's AEC3 over the local microphone capture, using the decoded remote audio as -/// the echo reference. -/// -/// The loudspeaker plays the remote audio, the microphone re-captures it, and the published -/// track would carry it back to everyone. Unity's Microphone path has no echo canceller -/// anywhere (capture never goes through a platform audio device module), so cancellation is -/// done here against the two streams already available in managed code: the FFI render frames -/// (far end, via ) and OnAudioFilterRead (near end). -/// -/// Threading: runs on the Unity audio thread and -/// on the FFI callback thread. Each owns its own pump, and -/// libwebrtc's APM is built for exactly that capture/render thread split. Nothing here touches -/// a Unity API from those threads; diagnostics use , not -/// UnityEngine.Time. -/// -/// Limitation: the far-end tap is not filtered by stream, so it assumes a SINGLE remote audio -/// stream. With several remote speakers the reference becomes the interleaving of all their -/// frames and AEC3 will not converge. The remote audio must also play through Unity -/// (AudioStream) — in PlatformAudio mode no FFI audio streams exist and nothing arrives. -/// -internal sealed class AcousticEchoCanceller : IDisposable -{ - private const int DiagnosticIntervalMs = 5000; - private const int SeedIntervalMs = 2000; - private const int SeedChangeThresholdMs = 5; - - private readonly AudioProcessingModule _apm; - private readonly ApmChunkPump _capturePump; - private readonly ApmChunkPump _renderPump; - private readonly Stopwatch _clock = new Stopwatch(); - - private long _nextDiagnosticMs; - private long _nextSeedMs; - private int _seededDelayMs = -1; - private int _farEndFrames; - private int _unsupportedRateWarned; - private bool _subscribed; - private bool _disposed; - - /// Processed 10 ms capture chunks, raised on the Unity audio thread. - public event ProcessedChunkHandler CaptureProcessed; - - private AcousticEchoCanceller(AudioProcessingModule apm) - { - _apm = apm; - _capturePump = new ApmChunkPump(_apm.ProcessStream, RaiseCaptureProcessed); - _renderPump = new ApmChunkPump(_apm.ProcessReverseStream); - } - - /// - /// Creates the canceller, or returns null when the FFI cannot hand out an APM handle — the - /// caller then publishes the unprocessed microphone rather than failing to publish at all. - /// - public static AcousticEchoCanceller TryCreate() - { - try - { - var apm = new AudioProcessingModule( - echoCancellerEnabled: true, - gainControllerEnabled: true, - highPassFilterEnabled: true, - noiseSuppressionEnabled: true); - - Debug.Log($"[AEC] APM created handle={apm.Handle}"); - return new AcousticEchoCanceller(apm); - } - catch (Exception e) - { - Debug.LogWarning($"[AEC] APM unavailable, publishing microphone unprocessed: {e.Message}"); - return null; - } - } - - public void Start() - { - if (_disposed || _subscribed) return; - - _capturePump.Reset(); - _renderPump.Reset(); - _clock.Restart(); - _nextDiagnosticMs = DiagnosticIntervalMs; - _nextSeedMs = SeedIntervalMs; - - // Forget the delay seeded before the last Stop(). A resume restarts the whole audio path, - // so the previous value describes an acoustic path that no longer exists — and carrying it - // over lets the SeedChangeThresholdMs guard in SeedStreamDelay silently skip the reseed - // below whenever the new estimate lands within 5 ms of the stale one. - _seededDelayMs = -1; - - FfiFrameObserver.AudioFrameReceived += OnFarEndFrame; - _subscribed = true; - - SeedStreamDelay(0d); - } - - public void Stop() - { - if (!_subscribed) return; - - FfiFrameObserver.AudioFrameReceived -= OnFarEndFrame; - _subscribed = false; - _clock.Reset(); - } - - /// - /// Feeds near-end capture. Unity audio thread. Returns false when the block cannot be - /// processed, and the caller must publish it unchanged — the APM needs a rate whose 10 ms - /// chunk is a whole number of samples, and device audio backends do run at odd rates. - /// - public bool TryPushCapture(float[] data, int channels, int sampleRate) - { - if (_disposed || data == null || channels <= 0 || sampleRate <= 0) return false; - - if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) - { - WarnUnsupportedRateOnce(sampleRate); - return false; - } - - _capturePump.Push(data, channels, sampleRate); - - var captureBlockMs = data.Length / (double)channels * 1000d / sampleRate; - MaybeReseed(captureBlockMs); - MaybeLogDiagnostics(channels, sampleRate); - return true; - } - - private void WarnUnsupportedRateOnce(int sampleRate) - { - if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; - - Debug.LogWarning( - $"[AEC] capture rate {sampleRate} has no whole-sample 10 ms chunk — " + - "echo cancellation disabled, publishing microphone unprocessed"); - } - - // The far-end DataPtr is valid for the duration of this callback ONLY; the pump copies out - // before doing anything else. - private void OnFarEndFrame(RawAudioFrame frame) - { - if (_disposed) return; - - Interlocked.Increment(ref _farEndFrames); - _renderPump.Push(frame.DataPtr, frame.SamplesPerChannel, frame.NumChannels, frame.SampleRate); - } - - private void RaiseCaptureProcessed(float[] data, int channels, int sampleRate) - { - CaptureProcessed?.Invoke(data, channels, sampleRate); - } - - // AVAudioSession reports zero latency until the session goes active, so the seed is - // re-evaluated on a slow cadence rather than only once at Start(). - private void MaybeReseed(double captureBlockMs) - { - if (!_clock.IsRunning) return; - - var elapsedMs = _clock.ElapsedMilliseconds; - if (elapsedMs < _nextSeedMs) return; - _nextSeedMs = elapsedMs + SeedIntervalMs; - - SeedStreamDelay(captureBlockMs); - } - - private void SeedStreamDelay(double captureBlockMs) - { - var delayMs = AudioProcessingDelaySeed.Estimate(captureBlockMs); - if (_seededDelayMs >= 0 && Math.Abs(delayMs - _seededDelayMs) < SeedChangeThresholdMs) return; - - var error = _apm.SetStreamDelayMs(delayMs); - if (error != null) - { - Debug.LogWarning($"[AEC] set_stream_delay_ms({delayMs}) failed: {error}"); - return; - } - - _seededDelayMs = delayMs; - Debug.Log($"[AEC] stream delay seeded to {delayMs}ms (captureBlock={captureBlockMs:F1}ms)"); - } - - // Reports the measured frame geometry both feeds are actually running at. Stopwatch, not - // UnityEngine.Time: this is the audio thread. - private void MaybeLogDiagnostics(int channels, int sampleRate) - { - if (!_clock.IsRunning) return; - - var elapsedMs = _clock.ElapsedMilliseconds; - if (elapsedMs < _nextDiagnosticMs) return; - _nextDiagnosticMs = elapsedMs + DiagnosticIntervalMs; - - Debug.Log( - $"[AEC] capture {channels}ch@{sampleRate} native={AudioProcessingModule.IsNativeSampleRate(sampleRate)} " + - $"chunks={_capturePump.ProcessedChunkCount} dropped={_capturePump.DroppedSamples} " + - $"failed={_capturePump.FailedChunkCount} err={_capturePump.LastError ?? "-"} | " + - $"render {_renderPump.Channels}ch@{_renderPump.SampleRate} frames={_farEndFrames} " + - $"chunks={_renderPump.ProcessedChunkCount} dropped={_renderPump.DroppedSamples} " + - $"failed={_renderPump.FailedChunkCount} err={_renderPump.LastError ?? "-"} | " + - $"delay={_seededDelayMs}ms"); - } - - public void Dispose() - { - if (_disposed) return; - - Stop(); - _disposed = true; - CaptureProcessed = null; - _capturePump.Dispose(); - _renderPump.Dispose(); - _apm.Dispose(); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs b/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs deleted file mode 100644 index 3ae50e5b..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecAudioProbe.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using UnityEngine; - -/// -/// Intercepts the microphone clip's audio on the Unity audio thread. -/// -/// Sample-side re-implementation of the SDK's AudioProbe, which is internal. Behaviour is -/// deliberately identical, including — without it the -/// microphone is played back through the local loudspeaker. -/// -internal sealed class AecAudioProbe : MonoBehaviour -{ - public delegate void OnAudioDelegate(float[] data, int channels, int sampleRate); - - public event OnAudioDelegate AudioRead; - - private int _sampleRate; - private volatile bool _clearAfterInvocation; - - public void ClearAfterInvocation() - { - _clearAfterInvocation = true; - } - - private void OnEnable() - { - OnAudioConfigurationChanged(false); - AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; - } - - private void OnDisable() - { - AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; - } - - private void OnAudioConfigurationChanged(bool deviceWasChanged) - { - _sampleRate = AudioSettings.outputSampleRate; - } - - private void OnAudioFilterRead(float[] data, int channels) - { - AudioRead?.Invoke(data, channels, _sampleRate); - if (_clearAfterInvocation) data.AsSpan().Clear(); - } - - private void OnDestroy() - { - AudioRead = null; - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs deleted file mode 100644 index 3f4a21f4..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneHost.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using UnityEngine; - -/// -/// Coroutine runner and application-pause relay for , attached -/// to the microphone GameObject. Stands in for the SDK's internal MonoBehaviourContext, -/// which sample code cannot reach. -/// -internal sealed class AecMicrophoneHost : MonoBehaviour -{ - public event Action Paused; - - private void OnApplicationPause(bool pause) - { - Paused?.Invoke(pause); - } - - private void OnDestroy() - { - Paused = null; - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs deleted file mode 100644 index c9f99ef3..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs +++ /dev/null @@ -1,304 +0,0 @@ -using System; -using System.Collections; -using LiveKit; -using UnityEngine; - -/// -/// Microphone capture source that runs AEC3 over the captured PCM before publishing it. -/// -/// The SDK's MicrophoneSource is sealed and its AudioProbe and -/// MonoBehaviourContext are internal, so the capture path is re-implemented here. Its -/// behaviours are load-bearing and preserved: clear-after-invocation (so the microphone is not -/// played back locally), the duplicate-component guard, the Microphone.GetPosition -/// readiness poll, and the pause/resume stop-restart cycle. -/// -/// The only behavioural difference is that carries APM-processed 10 ms -/// chunks instead of raw DSP blocks. When the APM is unavailable the raw blocks pass straight -/// through, so publishing never fails because of the canceller. -/// -internal sealed class AecMicrophoneSource : RtcAudioSource -{ - private readonly GameObject _sourceObject; - private readonly string _deviceName; - private readonly AecMicrophoneHost _host; - private readonly AcousticEchoCanceller _canceller; - - public override event Action AudioRead; - - private bool _disposed; - private bool _started; - - public bool EchoCancellationActive => _canceller != null; - - private AecMicrophoneSource( - string deviceName, - GameObject sourceObject, - AecMicrophoneHost host, - AcousticEchoCanceller canceller) - : base(RtcAudioSourceType.AudioSourceMicrophone) - { - _deviceName = deviceName; - _sourceObject = sourceObject; - _host = host; - _canceller = canceller; - - if (_canceller != null) _canceller.CaptureProcessed += OnProcessedAudio; - } - - /// - /// Builds the source. The base constructor configures the native source from Unity's current - /// output configuration, which is also the format OnAudioFilterRead delivers, so the - /// published track's metadata matches the capture geometry without further alignment. - /// - /// One of . - /// GameObject that hosts the AudioSource, probe and coroutine - /// runner. Must stay alive for the source's lifetime. - public static AecMicrophoneSource Create(string deviceName, GameObject sourceObject) - { - if (sourceObject == null) throw new ArgumentNullException(nameof(sourceObject)); - - var host = sourceObject.GetComponent(); - if (host == null) host = sourceObject.AddComponent(); - - return new AecMicrophoneSource(deviceName, sourceObject, host, AcousticEchoCanceller.TryCreate()); - } - - public override void Start() - { - base.Start(); - if (_started) return; - - if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) - throw new InvalidOperationException("Microphone access not authorized"); - - _host.Paused += OnApplicationPause; - _canceller?.Start(); - RunCoroutine(StartMicrophone()); - - _started = true; - } - - public override void Stop() - { - base.Stop(); - RunCoroutine(StopMicrophone()); - if (_host != null) _host.Paused -= OnApplicationPause; - _canceller?.Stop(); - _started = false; - } - - private IEnumerator StartMicrophone() - { - if (_sourceObject == null) - { - Debug.LogError("[AEC] microphone GameObject is null, cannot start"); - yield break; - } - - if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) - { - Debug.LogError("[AEC] microphone authorization lost"); - yield break; - } - - AudioClip clip = null; - try - { - clip = Microphone.Start( - _deviceName, - loop: true, - lengthSec: 1, - frequency: SupportedCaptureFrequency()); - } - catch (Exception e) - { - Debug.LogError($"[AEC] exception starting microphone: {e.Message}"); - yield break; - } - - if (clip == null) - { - Debug.LogError("[AEC] Microphone.Start returned null, audio session may not be ready"); - yield break; - } - - // Unity's Destroy is deferred, so a resume can land here while the previous pair is still - // alive. Duplicates would double every captured block into the APM. - var existingSource = _sourceObject.GetComponent(); - if (existingSource != null) UnityEngine.Object.DestroyImmediate(existingSource); - - var existingProbe = _sourceObject.GetComponent(); - if (existingProbe != null) - { - existingProbe.AudioRead -= OnCapturedAudio; - UnityEngine.Object.DestroyImmediate(existingProbe); - } - - var source = _sourceObject.AddComponent(); - source.clip = clip; - source.loop = true; - - var probe = _sourceObject.AddComponent(); - probe.ClearAfterInvocation(); - probe.AudioRead += OnCapturedAudio; - - const float timeout = 2f; - var elapsed = 0f; - while (Microphone.GetPosition(_deviceName) <= 0 && elapsed < timeout) - { - yield return new WaitForSeconds(0.05f); - elapsed += 0.05f; - } - - if (Microphone.GetPosition(_deviceName) <= 0) - { - Debug.LogError($"[AEC] microphone did not start producing data after {timeout}s"); - yield break; - } - - source.Play(); - Debug.Log($"[AEC] microphone '{_deviceName}' started at {clip.frequency}Hz, echo cancellation={EchoCancellationActive}"); - } - - // The requested rate is Unity's output rate, which the device may not accept as a capture - // rate. Clamping keeps Microphone.Start working; the DSP graph resamples the clip anyway, so - // OnAudioFilterRead still delivers the output rate either way. - private int SupportedCaptureFrequency() - { - var requested = AudioSettings.outputSampleRate; - Microphone.GetDeviceCaps(_deviceName, out var min, out var max); - - // Unity reports 0/0 when the device accepts any frequency. - if (min == 0 && max == 0) return requested; - - var clamped = Mathf.Clamp(requested, min, max); - if (clamped != requested) - Debug.Log($"[AEC] capture rate clamped {requested} -> {clamped} (device caps {min}-{max})"); - - return clamped; - } - - private IEnumerator StopMicrophone() - { - if (Microphone.IsRecording(_deviceName)) - Microphone.End(_deviceName); - - if (_sourceObject != null) - { - var probe = _sourceObject.GetComponent(); - if (probe != null) - { - probe.AudioRead -= OnCapturedAudio; - UnityEngine.Object.Destroy(probe); - } - - var source = _sourceObject.GetComponent(); - if (source != null) - UnityEngine.Object.Destroy(source); - } - - Debug.Log($"[AEC] microphone '{_deviceName}' stopped"); - yield return null; - } - - // Unity audio thread. A block the canceller cannot take is published unchanged rather than - // dropped — an un-cancelled participant beats a silent one. - private void OnCapturedAudio(float[] data, int channels, int sampleRate) - { - if (_canceller != null && _canceller.TryPushCapture(data, channels, sampleRate)) return; - - AudioRead?.Invoke(data, channels, sampleRate); - } - - // Unity audio thread, via the capture pump. - private void OnProcessedAudio(float[] data, int channels, int sampleRate) - { - AudioRead?.Invoke(data, channels, sampleRate); - } - - private void OnApplicationPause(bool pause) - { - if (!_started) return; - - if (pause) - { - // Backgrounded, release the audio resources — leaving them open trips - // AVAudioSession interruption errors (FigCaptureSourceRemote -17281). - _canceller?.Stop(); - RunCoroutine(StopMicrophone()); - } - else - { - RunCoroutine(RestartMicrophone()); - } - } - - private IEnumerator RestartMicrophone() - { - yield return StopMicrophone(); - - // After a resume the iOS audio session needs time to recover from interruption. Poll for - // actual readiness instead of guessing a delay. - yield return WaitForMicrophoneReady(); - - _canceller?.Start(); - yield return StartMicrophone(); - } - - private IEnumerator WaitForMicrophoneReady() - { - const float timeout = 2f; - var elapsed = 0f; - - while (Microphone.devices.Length == 0 && elapsed < timeout) - { - yield return new WaitForSeconds(0.05f); - elapsed += 0.05f; - } - - if (Microphone.devices.Length == 0) - { - Debug.LogError($"[AEC] microphone devices not available after {timeout}s"); - yield break; - } - - yield return null; - } - - // The host is a component on the (caller-owned) microphone GameObject. If that object is - // already gone — scene unload, app quit — drain the coroutine synchronously so Microphone.End - // and the component cleanup still run, as the SDK's MonoBehaviourContext does. - private void RunCoroutine(IEnumerator coroutine) - { - if (_host != null) - { - _host.StartCoroutine(coroutine); - return; - } - - while (coroutine.MoveNext()) - { - if (coroutine.Current is IEnumerator nested) - RunCoroutine(nested); - } - } - - protected override void Dispose(bool disposing) - { - if (!_disposed && disposing) Stop(); - _disposed = true; - - if (_canceller != null) - { - _canceller.CaptureProcessed -= OnProcessedAudio; - _canceller.Dispose(); - } - - base.Dispose(disposing); - } - - ~AecMicrophoneSource() - { - Dispose(false); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta deleted file mode 100644 index 83f422c2..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AecMicrophoneSource.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 582d502ba6935413ebc5cc0683c21059 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs deleted file mode 100644 index 7159b8d8..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using LiveKit; - -/// Processes one 10 ms interleaved int16 chunk in place at . -internal delegate string ApmChunkHandler(IntPtr dataPtr, int byteCount, int sampleRate, int channels); - -/// Receives one processed 10 ms chunk as interleaved floats. -internal delegate void ProcessedChunkHandler(float[] data, int channels, int sampleRate); - -/// -/// Re-chunks a variable-size PCM feed into the fixed 10 ms frames the APM requires, processes -/// each one in place, and optionally hands the result on. -/// -/// Neither feed is 10 ms natively: capture arrives in DSP-buffer-sized blocks (1024 frames -/// ≈ 21.3 ms at 48 kHz) and render frames arrive at whatever the decoder emits. Geometry is -/// taken from the incoming buffers, never from declared constants. -/// -/// Allocation-free once the format has settled. The chunk buffer stays pinned for the pump's -/// lifetime so the APM has a stable address to process in place. -/// -internal sealed class ApmChunkPump : IDisposable -{ - private readonly ApmChunkHandler _process; - private readonly ProcessedChunkHandler _onProcessed; - private readonly int _bufferedChunks; - - private PcmRingBuffer _ring; - private short[] _chunk; - private GCHandle _chunkPin; - private IntPtr _chunkPtr; - private int _chunkSamples; - private int _chunkBytes; - - private short[] _staging; - private float[] _processed; - - private int _sampleRate; - private int _channels; - private bool _disposed; - - public int SampleRate => _sampleRate; - public int Channels => _channels; - public int ChunkSamples => _chunkSamples; - public int ProcessedChunkCount { get; private set; } - public int FailedChunkCount { get; private set; } - public int DroppedSamples => _ring?.OverflowSamples ?? 0; - public string LastError { get; private set; } - - /// Processes one chunk in place; returns an error string or null. - /// Optional consumer of the processed chunk (capture side only). - /// - /// Ring capacity in 10 ms chunks. Sets the worst-case added latency, so keep it just large - /// enough to absorb one input block plus jitter. - /// - public ApmChunkPump(ApmChunkHandler process, ProcessedChunkHandler onProcessed = null, int bufferedChunks = 8) - { - if (bufferedChunks <= 0) throw new ArgumentOutOfRangeException(nameof(bufferedChunks)); - _process = process ?? throw new ArgumentNullException(nameof(process)); - _onProcessed = onProcessed; - _bufferedChunks = bufferedChunks; - } - - /// Feeds interleaved floats (Unity capture path). - public void Push(float[] data, int channels, int sampleRate) - { - if (_disposed || data == null || data.Length == 0) return; - if (!EnsureFormat(sampleRate, channels, data.Length)) return; - - if (_staging == null || _staging.Length < data.Length) _staging = new short[data.Length]; - for (var i = 0; i < data.Length; i++) _staging[i] = FloatToS16(data[i]); - - _ring.Write(_staging, 0, data.Length); - Drain(); - } - - /// Feeds interleaved int16 straight from a native buffer (FFI render path). - public void Push(IntPtr dataPtr, int samplesPerChannel, int channels, int sampleRate) - { - if (_disposed || dataPtr == IntPtr.Zero || samplesPerChannel <= 0 || channels <= 0) return; - - var total = samplesPerChannel * channels; - if (!EnsureFormat(sampleRate, channels, total)) return; - - if (_staging == null || _staging.Length < total) _staging = new short[total]; - Marshal.Copy(dataPtr, _staging, 0, total); - - _ring.Write(_staging, 0, total); - Drain(); - } - - public void Reset() => _ring?.Clear(); - - // An FFI failure must not escape: on the capture side this runs inside OnAudioFilterRead, and - // an exception there takes out Unity's audio callback. The unprocessed chunk is forwarded - // instead, so a broken APM degrades to no cancellation rather than to no audio. - private void Drain() - { - while (_ring.TryDrain(_chunk, _chunkSamples)) - { - try - { - var error = _process(_chunkPtr, _chunkBytes, _sampleRate, _channels); - if (error != null) LastError = error; - } - catch (Exception e) - { - LastError = e.Message; - FailedChunkCount++; - } - - ProcessedChunkCount++; - - if (_onProcessed == null) continue; - - for (var i = 0; i < _chunkSamples; i++) _processed[i] = _chunk[i] / 32768f; - _onProcessed(_processed, _channels, _sampleRate); - } - } - - private bool EnsureFormat(int sampleRate, int channels, int incomingSamples) - { - if (sampleRate <= 0 || channels <= 0) return false; - if (sampleRate == _sampleRate && channels == _channels) return true; - - var frameSize = AudioProcessingModule.FrameSizeFor(sampleRate); - if (frameSize <= 0) return false; - - ReleaseChunk(); - - _sampleRate = sampleRate; - _channels = channels; - _chunkSamples = frameSize * channels; - _chunkBytes = _chunkSamples * sizeof(short); - - _chunk = new short[_chunkSamples]; - _chunkPin = GCHandle.Alloc(_chunk, GCHandleType.Pinned); - _chunkPtr = _chunkPin.AddrOfPinnedObject(); - _processed = new float[_chunkSamples]; - - // Never smaller than one input block, or a large block would immediately overflow. - var capacity = Math.Max(_chunkSamples * _bufferedChunks, incomingSamples + _chunkSamples); - _ring = new PcmRingBuffer(capacity); - return true; - } - - private void ReleaseChunk() - { - if (_chunkPin.IsAllocated) _chunkPin.Free(); - _chunkPtr = IntPtr.Zero; - _chunk = null; - } - - public void Dispose() - { - if (_disposed) return; - _disposed = true; - ReleaseChunk(); - } - - private static short FloatToS16(float v) - { - v *= 32768f; - if (v > 32767f) v = 32767f; - else if (v < -32768f) v = -32768f; - return (short)(v + Math.Sign(v) * 0.5f); - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta deleted file mode 100644 index 99d2ca85..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/ApmChunkPump.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 3213055d7f9af4da4b525a2d8b9f9a1d -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs deleted file mode 100644 index 3a4cdeb2..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -/// -/// Computes the set_stream_delay_ms seed: -/// (t_render - t_analyze) + (t_process - t_capture). -/// -/// AEC3 runs its own correlation-based delay estimator -/// (use_external_delay_estimator = false), so this is a convergence hint, not a hard -/// alignment — but a wildly wrong value slows convergence. iOS sources the platform terms from -/// AVAudioSession (see Plugins/iOS/AudioSessionLatency.mm); Android has no -/// equivalent accessor, so it starts from the buffering this pipeline adds and lets AEC3 find -/// the rest. -/// -/// Known gap: the reference is tapped on the decoder side, so any playout queue between the -/// decoder and the loudspeaker (Unity's AudioStream ring buffer, ~30-200 ms) is not part of -/// this estimate. -/// -internal static class AudioProcessingDelaySeed -{ -#if UNITY_IOS && !UNITY_EDITOR - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionOutputLatency(); - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionInputLatency(); - [DllImport("__Internal")] private static extern double MeetSample_AudioSessionIOBufferDuration(); -#endif - - private const int MinDelayMs = 0; - private const int MaxDelayMs = 500; - - /// Seed used when no platform latency is readable (Android, or an inactive session). - private const int FallbackDelayMs = 60; - - /// - /// Measured duration of one OnAudioFilterRead block, which is how far behind - /// real-time the capture stream already is when it reaches the APM. - /// - public static int Estimate(double captureBlockMs) - { - var platformMs = PlatformLatencyMs(); - var seed = (int)Math.Round(platformMs + captureBlockMs); - return Math.Min(MaxDelayMs, Math.Max(MinDelayMs, seed)); - } - - public static double PlatformLatencyMs() - { -#if UNITY_IOS && !UNITY_EDITOR - try - { - var output = MeetSample_AudioSessionOutputLatency(); - var input = MeetSample_AudioSessionInputLatency(); - var ioBuffer = MeetSample_AudioSessionIOBufferDuration(); - if (output > 0d || input > 0d) - return (output + input + ioBuffer) * 1000d; - } - catch (Exception) - { - // Session not yet active; fall through to the platform-agnostic seed. - } - return FallbackDelayMs; -#else - return FallbackDelayMs; -#endif - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta deleted file mode 100644 index 086e91bd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingDelaySeed.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 9636ad91efd914976b3ed12df25e7e8c -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs deleted file mode 100644 index 859260bd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using LiveKit; -using UnityEngine; - -/// -/// Go/no-go check for the FFI , runnable without joining a -/// room. Exercises the full call shape: create a handle, seed a delay, and process one 10 ms -/// chunk in each direction. The APM is compiled into each platform's FFI binary separately, so -/// a per-platform surprise is far cheaper to find here than after the pipeline is wired. -/// -internal static class AudioProcessingSmokeTest -{ - public static string Run() - { - var rate = AudioSettings.outputSampleRate > 0 ? AudioSettings.outputSampleRate : 48000; - var frameSize = AudioProcessingModule.FrameSizeFor(rate); - - AudioProcessingModule apm; - try - { - apm = new AudioProcessingModule( - echoCancellerEnabled: true, - gainControllerEnabled: false, - highPassFilterEnabled: false, - noiseSuppressionEnabled: false); - } - catch (Exception e) - { - return $"FAIL create_apm: {e.GetType().Name}: {e.Message}"; - } - - using (apm) - { - var delayError = apm.SetStreamDelayMs(100); - if (delayError != null) return $"FAIL set_stream_delay_ms: {delayError}"; - - var chunk = new short[frameSize]; - var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); - try - { - var ptr = pin.AddrOfPinnedObject(); - var bytes = chunk.Length * sizeof(short); - - var reverseError = apm.ProcessReverseStream(ptr, bytes, rate, 1); - if (reverseError != null) return $"FAIL process_reverse_stream: {reverseError}"; - - var processError = apm.ProcessStream(ptr, bytes, rate, 1); - if (processError != null) return $"FAIL process_stream: {processError}"; - } - finally - { - pin.Free(); - } - - return $"PASS handle={apm.Handle} rate={rate} frameSize={frameSize} " + - $"nativeRate={AudioProcessingModule.IsNativeSampleRate(rate)} " + - $"delaySeed={AudioProcessingDelaySeed.Estimate(0d)}ms"; - } - } -} diff --git a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta b/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta deleted file mode 100644 index cf3748bf..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/AudioProcessingSmokeTest.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 78f2bfe939fd54d97b797b6b28398e88 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs b/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs deleted file mode 100644 index f4b3e8fd..00000000 --- a/Samples~/Meet/Assets/Runtime/Audio/PcmRingBuffer.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; - -/// -/// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. -/// -/// Allocation-free after construction — both feeds run on audio-priority threads. Sized in -/// samples (interleaved, i.e. frames × channels), not frames. -/// -/// Single producer / single consumer per direction; the near-end and far-end feeds each own -/// their own instance, so no synchronisation is needed inside. -/// -internal sealed class PcmRingBuffer -{ - private readonly short[] _buffer; - private int _readIndex; - private int _writeIndex; - private int _count; - - /// Samples dropped because the buffer was full, since construction. - public int OverflowSamples { get; private set; } - - public int Capacity => _buffer.Length; - public int Available => _count; - - public PcmRingBuffer(int capacitySamples) - { - if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); - _buffer = new short[capacitySamples]; - } - - /// - /// Appends samples. When the buffer is full the OLDEST samples are - /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of - /// alignment with the capture stream. - /// - public void Write(short[] source, int offset, int count) - { - if (source == null) throw new ArgumentNullException(nameof(source)); - if (offset < 0 || count < 0 || offset + count > source.Length) - throw new ArgumentOutOfRangeException(nameof(count)); - - if (count >= _buffer.Length) - { - OverflowSamples += _count + count - _buffer.Length; - offset += count - _buffer.Length; - count = _buffer.Length; - _readIndex = 0; - _writeIndex = 0; - _count = 0; - } - else - { - var free = _buffer.Length - _count; - if (count > free) Discard(count - free); - } - - for (var i = 0; i < count; i++) - { - _buffer[_writeIndex] = source[offset + i]; - _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; - } - - _count += count; - } - - /// - /// Copies exactly samples into and - /// consumes them. Returns false and consumes nothing when fewer are available. - /// - public bool TryDrain(short[] destination, int count) - { - if (destination == null) throw new ArgumentNullException(nameof(destination)); - if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); - if (_count < count) return false; - - for (var i = 0; i < count; i++) - { - destination[i] = _buffer[_readIndex]; - _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; - } - - _count -= count; - return true; - } - - public void Clear() - { - _readIndex = 0; - _writeIndex = 0; - _count = 0; - } - - private void Discard(int count) - { - if (count > _count) count = _count; - _readIndex = (_readIndex + count) % _buffer.Length; - _count -= count; - OverflowSamples += count; - } -} diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index c0bf6c0a..b9893fbc 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -13,9 +13,9 @@ /// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// Gives more control over audio processing. Optionally runs libwebrtc's AEC3 over the -/// captured audio (), using the decoded remote audio frames -/// as the echo reference; without it there is no echo cancellation in this mode. +/// Gives more control over audio processing. The same AEC/NS/AGC toggles apply: the SDK runs +/// libwebrtc's audio processing over the Microphone capture, with the mix Unity plays as the +/// echo reference (see ). /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -36,22 +36,18 @@ public class MeetManager : MonoBehaviour "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")] [SerializeField] private bool usePlatformAudio = true; - [Header("Audio Processing (PlatformAudio only)")] - [Tooltip("Enable echo cancellation to remove echo from speaker playback.")] + [Header("Audio Processing")] + [Tooltip("Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")] [SerializeField] private bool echoCancellation = true; [Tooltip("Enable noise suppression to remove background noise.")] [SerializeField] private bool noiseSuppression = true; [Tooltip("Enable auto gain control to normalize audio levels.")] [SerializeField] private bool autoGainControl = true; - [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] + [Tooltip("PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; [Header("Unity Audio (PlatformAudio off)")] - [Tooltip("Run libwebrtc's AEC3 (the FFI AudioProcessingModule) over Unity microphone capture, " + - "using the decoded remote audio frames as the echo reference. Assumes a single remote " + - "audio stream: with several remote speakers the reference is the interleaving of all of " + - "them and the canceller will not converge.")] - [SerializeField] private bool unityEchoCancellation = true; [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] @@ -104,8 +100,6 @@ private void Start() if (usePlatformAudio) InitializePlatformAudio(); - else if (unityEchoCancellation) - Debug.Log($"AEC smoke test: {AudioProcessingSmokeTest.Run()}"); } private void InitializePlatformAudio() @@ -622,7 +616,7 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log($"Publishing microphone using Unity Microphone API (AEC3: {unityEchoCancellation})"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); @@ -630,14 +624,18 @@ private IEnumerator PublishLocalMicrophoneUnity() var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - // AecMicrophoneSource re-implements MicrophoneSource with an APM stage in between: - // captured blocks go through AEC3 (reference = decoded remote frames) before they reach - // the track. If the APM cannot be created it publishes the raw microphone instead. - RtcAudioSource rtcSource; - if (unityEchoCancellation) - rtcSource = AecMicrophoneSource.Create(Microphone.devices[0], audioObject); - else - rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // With options, MicrophoneSource runs libwebrtc's audio processing over the capture. Echo + // cancellation takes its reference from the mix Unity plays (the SDK attaches a + // PlayoutReference to the AudioListener), so it covers every remote AudioStream and the + // app's own audio. If the module cannot be created the source publishes the raw microphone. + var processing = new AudioProcessingOptions + { + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl, + HighPassFilter = true + }; + var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -652,8 +650,8 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { - // Dispose before destroying the host object so the source (and, for AEC, its APM - // handle) is released now rather than by the finalizer. + // Dispose before destroying the host object so the source and its processing module + // are released now rather than by the finalizer. rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; @@ -665,9 +663,24 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log(rtcSource is AecMicrophoneSource { EchoCancellationActive: true } - ? "Microphone published via Unity Microphone API (AEC3 active)" - : "Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource.AudioProcessingEnabled + ? "Microphone published via Unity Microphone API (audio processing active)" + : "Microphone published via Unity Microphone API (no audio processing)"); + + if (rtcSource.AudioProcessingEnabled) + StartCoroutine(LogAudioProcessingStats(rtcSource)); + } + + // Periodic snapshot of the processing stage while the Unity microphone is published: confirms + // the playout reference is attached and that capture and reference chunks are flowing. + private IEnumerator LogAudioProcessingStats(RtcAudioSource source) + { + while (_microphoneActive && ReferenceEquals(_localRtcAudioSource, source)) + { + yield return new WaitForSeconds(5f); + if (!_microphoneActive || !ReferenceEquals(_localRtcAudioSource, source)) yield break; + Debug.Log($"Audio processing: {source.AudioProcessingStats}"); + } } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 3122a7c1..3e03e2d1 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -907,7 +907,6 @@ MonoBehaviour: noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 - unityEchoCancellation: 1 remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: @@ -921,7 +920,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: 6d52b5cb4c971436098df568ef2e2c67, type: 2} + _config: {fileID: 11400000, guid: aae6c4b1158ca4c929c5d84962c95f91, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs new file mode 100644 index 00000000..0295dfbc --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -0,0 +1,155 @@ +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + /// + /// Pure-managed tests for the Unity-audio processing helpers: the 10 ms re-chunking, sample + /// conversion, rate rules and the delay hint. No FFI, no audio device, so they always run. + /// + public class AudioProcessingTests + { + [Test] + public void PcmRingBuffer_DrainsInWriteOrder() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2, 3, 4, 5 }, 0, 5); + + var dest = new short[3]; + Assert.IsTrue(ring.TryDrain(dest, 3)); + Assert.AreEqual(new short[] { 1, 2, 3 }, dest); + Assert.AreEqual(2, ring.Available); + + var rest = new short[2]; + Assert.IsTrue(ring.TryDrain(rest, 2)); + Assert.AreEqual(new short[] { 4, 5 }, rest); + Assert.AreEqual(0, ring.Available); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_TryDrain_WithoutEnoughSamples_ConsumesNothing() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2 }, 0, 2); + + Assert.IsFalse(ring.TryDrain(new short[3], 3)); + Assert.AreEqual(2, ring.Available); + } + + [Test] + public void PcmRingBuffer_WhenFull_DropsOldest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Write(new short[] { 4, 5 }, 0, 2); + + Assert.AreEqual(1, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 2, 3, 4, 5 }, dest); + } + + [Test] + public void PcmRingBuffer_WriteLargerThanCapacity_KeepsNewest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2 }, 0, 2); + ring.Write(new short[] { 3, 4, 5, 6, 7 }, 0, 5); + + Assert.AreEqual(3, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 4, 5, 6, 7 }, dest); + } + + [Test] + public void PcmRingBuffer_WrapsAround() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + Assert.IsTrue(ring.TryDrain(new short[2], 2)); + ring.Write(new short[] { 4, 5, 6 }, 0, 3); + + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 3, 4, 5, 6 }, dest); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_Clear_Empties() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Clear(); + + Assert.AreEqual(0, ring.Available); + Assert.IsFalse(ring.TryDrain(new short[1], 1)); + } + + [Test] + public void PcmConvert_FloatToS16_ClampsAndRounds() + { + Assert.AreEqual(0, PcmConvert.FloatToS16(0f)); + Assert.AreEqual(16384, PcmConvert.FloatToS16(0.5f)); + Assert.AreEqual(-16384, PcmConvert.FloatToS16(-0.5f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(1f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-1f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(3f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-3f)); + } + + [Test] + public void AudioProcessingModule_SupportedApiRates_NeedWholeSampleChunks() + { + // The Rust side asserts on a frame that is not a whole multiple of 10 ms, so a rate whose + // 10 ms chunk is fractional must be refused up front. + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(48000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(44100)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(24000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(16000)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(22050)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(11025)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(0)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(-48000)); + } + + [Test] + public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() + { + Assert.AreEqual(480, AudioProcessingModule.FrameSizeFor(48000)); + Assert.AreEqual(441, AudioProcessingModule.FrameSizeFor(44100)); + Assert.AreEqual(240, AudioProcessingModule.FrameSizeFor(24000)); + Assert.IsTrue(AudioProcessingModule.IsNativeSampleRate(48000)); + Assert.IsFalse(AudioProcessingModule.IsNativeSampleRate(44100)); + } + + [Test] + public void AudioProcessingOptions_Default_EnablesHighPassFilter_AndReportsProcessing() + { + Assert.IsTrue(AudioProcessingOptions.Default.HighPassFilter); + Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); + + // An all-false struct means "no processing"; PreferHardware alone is not a stage. + Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); + Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { HighPassFilter = true }.AnyProcessingEnabled); + } + + [Test] + public void DelayHint_SumsQueueDeviceAndMicrophoneTerms() + { + // 1024 frames at 48 kHz = 21.33 ms per block; two queued blocks + 30 ms device + 50 ms + // microphone read-behind = 122.67 ms. + Assert.AreEqual(123, AudioProcessingDelayHint.EstimateMs(1024, 48000, 30)); + Assert.AreEqual(AudioProcessingDelayHint.MicrophoneReadBehindMs, AudioProcessingDelayHint.EstimateMs(0, 0, 0)); + } + + [Test] + public void DelayHint_ClampsToRange() + { + Assert.AreEqual(AudioProcessingDelayHint.MaxDelayMs, AudioProcessingDelayHint.EstimateMs(48000, 48000, 1000)); + Assert.AreEqual(AudioProcessingDelayHint.MinDelayMs, AudioProcessingDelayHint.EstimateMs(0, 0, -1000)); + } + } +} diff --git a/Tests/EditMode/AudioProcessingTests.cs.meta b/Tests/EditMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..80c9b2e9 --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e9ce682bd85ef402eb8ce57e87d70231 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/EditMode/PlatformAudioTests.cs b/Tests/EditMode/PlatformAudioTests.cs index b7608ac7..8c4bca40 100644 --- a/Tests/EditMode/PlatformAudioTests.cs +++ b/Tests/EditMode/PlatformAudioTests.cs @@ -18,6 +18,7 @@ public void AudioProcessingOptions_Default_EnablesProcessingAndHardware() Assert.IsTrue(options.EchoCancellation, "AEC should be enabled by default"); Assert.IsTrue(options.NoiseSuppression, "NS should be enabled by default"); Assert.IsTrue(options.AutoGainControl, "AGC should be enabled by default"); + Assert.IsTrue(options.HighPassFilter, "HPF should be enabled by default"); // Unlike the C++ defaults (prefer_hardware == false), the Unity default prefers // hardware processing (e.g. iOS VPIO) for lower latency. Assert.IsTrue(options.PreferHardware, "Unity default prefers hardware processing"); diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs new file mode 100644 index 00000000..f132e77e --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using NUnit.Framework; +using Unity.Collections; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// Tests for the Unity-audio processing stage. They need the FFI for the module and, for the + /// echo test, Unity's audio thread. No LiveKit server. + /// + class AudioProcessingTests + { + [Test] + public void AudioProcessingModule_AcceptsTenMillisecondChunks() + { + using var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + const int rate = 48000; + const int channels = 2; + var chunk = new short[AudioProcessingModule.FrameSizeFor(rate) * channels]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + Assert.IsNull(apm.ProcessReverseStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.ProcessStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.SetStreamDelayMs(80)); + } + finally + { + pin.Free(); + } + } + + [UnityTest] + public IEnumerator RtcAudioSource_WithProcessing_CapturesInTenMillisecondChunks() + { + using var source = new PushAudioSource(AudioProcessingOptions.Default); + Assert.IsTrue(source.AudioProcessingEnabled, "module creation failed"); + + var rate = (int)source._expectedSampleRate; + var channels = (int)source._expectedChannels; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + source.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + source.Push(block, channels, rate); + + // Let the capture callbacks return before disposing. + yield return new WaitForSeconds(0.2f); + + var stats = source.AudioProcessingStats; + var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); + Assert.IsTrue(stats.Active, stats.ToString()); + Assert.AreEqual(0, stats.FailedChunks, stats.LastError); + Assert.That(stats.CaptureChunks, Is.InRange(expectedChunks - 1, expectedChunks), stats.ToString()); + Assert.AreEqual(0, stats.DroppedCaptureSamples, stats.ToString()); + + source.Stop(); + } + + /// + /// End-to-end check of the canceller without hardware: the listener hears a noise source, + /// and a second source plays the same noise 120 ms later, probed as "microphone" and then + /// cleared so the mix contains only the far end. The capture is therefore a pure delayed + /// echo of the playout reference, which AEC3 must learn to remove. + /// + [UnityTest] + public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() + { + var rate = AudioSettings.outputSampleRate; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var listenerGo = new GameObject("AecTestListener"); + listenerGo.AddComponent(); + + var clip = NoiseClip(rate, seconds: 2f, seed: 1234, amplitude: 0.3f); + + var farGo = new GameObject("AecTestFarEnd"); + var far = farGo.AddComponent(); + far.clip = clip; + far.loop = true; + + var nearGo = new GameObject("AecTestNearEnd"); + var near = nearGo.AddComponent(); + near.clip = clip; + near.loop = true; + var probe = nearGo.AddComponent(); + probe.ClearAfterInvocation(); + + var meter = new EchoMeter(); + var processor = new AudioProcessor( + new AudioProcessingOptions { EchoCancellation = true, HighPassFilter = true }, + meter.OnProcessed); + probe.AudioRead += (data, channels, sampleRate) => + { + meter.OnRaw(data); + processor.TryProcessCapture(data, channels, sampleRate); + }; + processor.Start(); + + var startTime = AudioSettings.dspTime + 0.2; + far.PlayScheduled(startTime); + near.PlayScheduled(startTime + 0.12); + + AudioProcessingStats stats; + float rawRms, processedRms; + try + { + yield return new WaitForSeconds(1f); + if (meter.RawBlocks == 0) + Assert.Ignore("Unity's audio thread delivered no capture callbacks (no audio device?)"); + Assert.IsTrue(PlayoutReference.IsAttached, "reference not attached to the listener"); + + // Convergence time, then a clean measurement window. + yield return new WaitForSeconds(3f); + meter.ResetWindow(); + yield return new WaitForSeconds(1.5f); + + (rawRms, processedRms) = meter.Window(); + stats = processor.GetStats(); + } + finally + { + processor.Dispose(); + UnityEngine.Object.Destroy(farGo); + UnityEngine.Object.Destroy(nearGo); + UnityEngine.Object.Destroy(listenerGo); + } + + Assert.Greater(rawRms, 0.01f, "near-end source produced no signal"); + Assert.AreEqual(0, stats.FailedChunks, stats.ToString()); + Assert.Greater(stats.ReferenceChunks, 0, stats.ToString()); + + var attenuationDb = 20f * Mathf.Log10(rawRms / Mathf.Max(processedRms, 1e-6f)); + Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB ({stats})"); + Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB; {stats}"); + } + + private static AudioClip NoiseClip(int sampleRate, float seconds, int seed, float amplitude) + { + var samples = (int)(sampleRate * seconds); + var data = new float[samples]; + var random = new System.Random(seed); + for (int i = 0; i < samples; i++) + data[i] = amplitude * (float)(random.NextDouble() * 2.0 - 1.0); + + var clip = AudioClip.Create("AecTestNoise", samples, 1, sampleRate, false); + clip.SetData(data, 0); + return clip; + } + + private sealed class PushAudioSource : RtcAudioSource + { + public override event Action AudioRead; + + public PushAudioSource(AudioProcessingOptions options) + : base(RtcAudioSourceType.AudioSourceMicrophone, options) { } + + public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); + } + + // Accumulates energy of the raw near end and of the processed output. Both callbacks run on + // the Unity audio thread; the sink owns and disposes the frames it is handed. + private sealed class EchoMeter + { + private readonly object _lock = new object(); + private double _rawSum; + private long _rawCount; + private double _processedSum; + private long _processedCount; + private int _rawBlocks; + + public int RawBlocks { get { lock (_lock) return _rawBlocks; } } + + public void OnRaw(float[] data) + { + double sum = 0; + for (int i = 0; i < data.Length; i++) sum += data[i] * data[i]; + lock (_lock) + { + _rawSum += sum; + _rawCount += data.Length; + _rawBlocks++; + } + } + + public void OnProcessed(NativeArray frame, int channels, int sampleRate) + { + double sum = 0; + var length = frame.Length; + for (int i = 0; i < length; i++) + { + var v = frame[i] / 32768.0; + sum += v * v; + } + frame.Dispose(); + lock (_lock) + { + _processedSum += sum; + _processedCount += length; + } + } + + public void ResetWindow() + { + lock (_lock) + { + _rawSum = 0; + _rawCount = 0; + _processedSum = 0; + _processedCount = 0; + } + } + + public (float raw, float processed) Window() + { + lock (_lock) return (Rms(_rawSum, _rawCount), Rms(_processedSum, _processedCount)); + } + + private static float Rms(double sum, long count) => count == 0 ? 0f : (float)Math.Sqrt(sum / count); + } + } +} diff --git a/Tests/PlayMode/AudioProcessingTests.cs.meta b/Tests/PlayMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..4b6b1e59 --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b22944e23e00341eb8cd913bce198b4e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From abeb06dcc3b544a33dd2e78e063a64af31df1265 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:20:29 +0200 Subject: [PATCH 03/11] Created subfolder for audio processing --- Runtime/Scripts/Audio/Processing.meta | 8 ++++++++ .../Audio/{ => Processing}/AudioProcessingDelayHint.cs | 0 .../{ => Processing}/AudioProcessingDelayHint.cs.meta | 0 .../Audio/{ => Processing}/AudioProcessingModule.cs | 0 .../Audio/{ => Processing}/AudioProcessingModule.cs.meta | 0 .../Audio/{ => Processing}/AudioProcessingStats.cs | 0 .../Audio/{ => Processing}/AudioProcessingStats.cs.meta | 0 Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs | 0 .../Scripts/Audio/{ => Processing}/AudioProcessor.cs.meta | 0 Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs | 0 .../Scripts/Audio/{ => Processing}/PcmRingBuffer.cs.meta | 0 .../Scripts/Audio/{ => Processing}/PlayoutReference.cs | 0 .../Audio/{ => Processing}/PlayoutReference.cs.meta | 0 13 files changed, 8 insertions(+) create mode 100644 Runtime/Scripts/Audio/Processing.meta rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingDelayHint.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingDelayHint.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingModule.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingModule.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingStats.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessingStats.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/AudioProcessor.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/PcmRingBuffer.cs.meta (100%) rename Runtime/Scripts/Audio/{ => Processing}/PlayoutReference.cs (100%) rename Runtime/Scripts/Audio/{ => Processing}/PlayoutReference.cs.meta (100%) diff --git a/Runtime/Scripts/Audio/Processing.meta b/Runtime/Scripts/Audio/Processing.meta new file mode 100644 index 00000000..180519b6 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: afb866c8c71a34134bf00d5cd4ee1778 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingDelayHint.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingDelayHint.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingModule.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingModule.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingStats.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs diff --git a/Runtime/Scripts/Audio/AudioProcessingStats.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessingStats.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessor.cs rename to Runtime/Scripts/Audio/Processing/AudioProcessor.cs diff --git a/Runtime/Scripts/Audio/AudioProcessor.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/AudioProcessor.cs.meta rename to Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs similarity index 100% rename from Runtime/Scripts/Audio/PcmRingBuffer.cs rename to Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs diff --git a/Runtime/Scripts/Audio/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/PcmRingBuffer.cs.meta rename to Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs similarity index 100% rename from Runtime/Scripts/Audio/PlayoutReference.cs rename to Runtime/Scripts/Audio/Processing/PlayoutReference.cs diff --git a/Runtime/Scripts/Audio/PlayoutReference.cs.meta b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta similarity index 100% rename from Runtime/Scripts/Audio/PlayoutReference.cs.meta rename to Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta From be5d9c4651f43590b47807ba699baeeae05b63df Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:53:33 +0200 Subject: [PATCH 04/11] Drop the HighPassFilter option and make AudioProcessingModule internal libwebrtc instantiates the high-pass filter whenever AEC or NS is enabled (echo_canceller.enforce_high_pass_filtering defaults to true) and the platform ADM path always runs it, so the flag only ever mattered with AEC and NS both off. Hardcode it on for the Unity-audio path so both paths match, and keep the FFI wrapper off the public surface: sources configure it through AudioProcessingOptions. Co-Authored-By: Claude Fable 5.1 --- README.md | 3 +-- Runtime/Scripts/Audio/PlatformAudioSource.cs | 11 ++--------- .../Scripts/Audio/Processing/AudioProcessingModule.cs | 2 +- Runtime/Scripts/Audio/Processing/AudioProcessor.cs | 5 ++++- Samples~/Meet/Assets/Runtime/MeetManager.cs | 3 +-- Tests/EditMode/AudioProcessingTests.cs | 5 ++--- Tests/EditMode/PlatformAudioTests.cs | 1 - Tests/PlayMode/AudioProcessingTests.cs | 2 +- 8 files changed, 12 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ce91509a..c4c78532 100644 --- a/README.md +++ b/README.md @@ -320,8 +320,7 @@ var processing = new AudioProcessingOptions { EchoCancellation = true, NoiseSuppression = true, - AutoGainControl = true, - HighPassFilter = true + AutoGainControl = true }; var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); ``` diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 5a7066fc..119adbf1 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -9,7 +9,7 @@ namespace LiveKit /// /// Options for libwebrtc's audio processing. Used by , where /// the ADM applies them, and by Unity-audio sources such as - /// created with options, where the SDK runs the over the + /// created with options, where the SDK runs libwebrtc's audio processing module over the /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions @@ -21,12 +21,6 @@ public struct AudioProcessingOptions /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; /// - /// Enable the high-pass filter, which removes DC offset and low-frequency rumble ahead of - /// the other stages. Unity-audio sources only; ignores it. - /// Default: true. - /// - public bool HighPassFilter; - /// /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. /// only. Default: true. /// @@ -40,12 +34,11 @@ public struct AudioProcessingOptions EchoCancellation = true, NoiseSuppression = true, AutoGainControl = true, - HighPassFilter = true, PreferHardware = true }; /// Whether any stage of the Unity-audio processing pipeline is enabled. - internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl || HighPassFilter; + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl; } /// diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs index d83f83be..5e264a2b 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs @@ -21,7 +21,7 @@ namespace LiveKit /// the native module is internally synchronised for exactly that split, and the SDK's request /// plumbing is safe to use from both. /// - public sealed class AudioProcessingModule : IDisposable + internal sealed class AudioProcessingModule : IDisposable { /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. public const int ChunkSizeMs = 10; diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index f9d8d34f..ca72566c 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -87,10 +87,13 @@ public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) { _sink = sink ?? throw new ArgumentNullException(nameof(sink)); _echoCancellation = options.EchoCancellation; + // The high-pass filter is not exposed as an option: libwebrtc instantiates it anyway + // whenever AEC or NS is on, and the platform ADM path always runs it, so both paths + // stay identical. _apm = new AudioProcessingModule( echoCancellerEnabled: options.EchoCancellation, gainControllerEnabled: options.AutoGainControl, - highPassFilterEnabled: options.HighPassFilter, + highPassFilterEnabled: true, noiseSuppressionEnabled: options.NoiseSuppression); } diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index b9893fbc..29314dfa 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -632,8 +632,7 @@ private IEnumerator PublishLocalMicrophoneUnity() { EchoCancellation = echoCancellation, NoiseSuppression = noiseSuppression, - AutoGainControl = autoGainControl, - HighPassFilter = true + AutoGainControl = autoGainControl }; var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs index 0295dfbc..4c29ba3b 100644 --- a/Tests/EditMode/AudioProcessingTests.cs +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -125,15 +125,14 @@ public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() } [Test] - public void AudioProcessingOptions_Default_EnablesHighPassFilter_AndReportsProcessing() + public void AudioProcessingOptions_AnyProcessingEnabled_TracksTheProcessingStages() { - Assert.IsTrue(AudioProcessingOptions.Default.HighPassFilter); Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); // An all-false struct means "no processing"; PreferHardware alone is not a stage. Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); - Assert.IsTrue(new AudioProcessingOptions { HighPassFilter = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { NoiseSuppression = true }.AnyProcessingEnabled); } [Test] diff --git a/Tests/EditMode/PlatformAudioTests.cs b/Tests/EditMode/PlatformAudioTests.cs index 8c4bca40..b7608ac7 100644 --- a/Tests/EditMode/PlatformAudioTests.cs +++ b/Tests/EditMode/PlatformAudioTests.cs @@ -18,7 +18,6 @@ public void AudioProcessingOptions_Default_EnablesProcessingAndHardware() Assert.IsTrue(options.EchoCancellation, "AEC should be enabled by default"); Assert.IsTrue(options.NoiseSuppression, "NS should be enabled by default"); Assert.IsTrue(options.AutoGainControl, "AGC should be enabled by default"); - Assert.IsTrue(options.HighPassFilter, "HPF should be enabled by default"); // Unlike the C++ defaults (prefer_hardware == false), the Unity default prefers // hardware processing (e.g. iOS VPIO) for lower latency. Assert.IsTrue(options.PreferHardware, "Unity default prefers hardware processing"); diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs index f132e77e..5e5f5c73 100644 --- a/Tests/PlayMode/AudioProcessingTests.cs +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -107,7 +107,7 @@ public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() var meter = new EchoMeter(); var processor = new AudioProcessor( - new AudioProcessingOptions { EchoCancellation = true, HighPassFilter = true }, + new AudioProcessingOptions { EchoCancellation = true }, meter.OnProcessed); probe.AudioRead += (data, channels, sampleRate) => { From fd508485a0252ac20fd447ffeb6785ded5d6a991 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:03:11 +0200 Subject: [PATCH 05/11] Drop AudioProcessingStats The struct and RtcAudioSource.AudioProcessingStats only mirrored counters the processor kept for diagnostics. Nothing in the processing path read them, and the only consumers were Meet's periodic log and two test assertions. The one signal that would otherwise go silent, the module rejecting a chunk, now warns once from the main thread the same way the unsupported-rate bypass does. Tests: the chunk count moves to the AudioProcessor level through a counting sink; the source-level test keeps the push path covered. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- Runtime/Scripts/Audio/MicrophoneSource.cs | 2 +- .../Audio/Processing/AudioProcessingStats.cs | 87 ------------------- .../Processing/AudioProcessingStats.cs.meta | 11 --- .../Audio/Processing/AudioProcessor.cs | 55 +++++------- Runtime/Scripts/Audio/RtcAudioSource.cs | 3 - Samples~/Meet/Assets/Runtime/MeetManager.cs | 15 ---- Tests/PlayMode/AudioProcessingTests.cs | 69 ++++++++++++--- 8 files changed, 78 insertions(+), 166 deletions(-) delete mode 100644 Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs delete mode 100644 Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta diff --git a/README.md b/README.md index c4c78532..5b9114ce 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,7 @@ var processing = new AudioProcessingOptions var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); ``` -Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. `rtcSource.AudioProcessingStats` reports whether the reference is attached and how many chunks flowed. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. +Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. #### Unity Audio Output diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 9f88ad38..48f42868 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -54,7 +54,7 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu /// Which stages to enable. With /// the SDK attaches a to the active to obtain the /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is - /// bypassed with a warning. See for diagnostics. + /// bypassed with a warning. public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing) : base(RtcAudioSourceType.AudioSourceMicrophone, processing) { diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs deleted file mode 100644 index 0362a81c..00000000 --- a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs +++ /dev/null @@ -1,87 +0,0 @@ -namespace LiveKit -{ - /// - /// Counters from the audio processing stage of an that was - /// created with . A snapshot; read it from any thread via - /// . - /// - public readonly struct AudioProcessingStats - { - /// - /// True while capture is actually being processed. False when the source has no processing - /// stage, is stopped, or bypasses processing because Unity's output sample rate has no - /// whole-sample 10 ms chunk (it is not a multiple of 100 Hz). - /// - public readonly bool Active; - - /// - /// True while a on an enabled AudioListener is feeding - /// the far-end reference. Without it echo cancellation has nothing to cancel; noise - /// suppression, gain control and the high-pass filter still run. - /// - public readonly bool ReferenceAttached; - - /// Format of the capture feed, as delivered by Unity's audio graph. - public readonly int CaptureSampleRate; - public readonly int CaptureChannels; - - /// Format of the playout reference feed. - public readonly int ReferenceSampleRate; - public readonly int ReferenceChannels; - - /// 10 ms capture chunks run through the module. - public readonly long CaptureChunks; - - /// 10 ms reference chunks run through the module. - public readonly long ReferenceChunks; - - /// Samples discarded because a feed outran its buffer. Non-zero means the audio thread stalled. - public readonly long DroppedCaptureSamples; - public readonly long DroppedReferenceSamples; - - /// Chunks the module rejected; those capture chunks were published unprocessed. - public readonly long FailedChunks; - - /// Most recent module error, or null. - public readonly string LastError; - - /// Render-to-capture delay hint last handed to the module, or -1 if none yet. - public readonly int StreamDelayHintMs; - - public AudioProcessingStats( - bool active, - bool referenceAttached, - int captureSampleRate, - int captureChannels, - int referenceSampleRate, - int referenceChannels, - long captureChunks, - long referenceChunks, - long droppedCaptureSamples, - long droppedReferenceSamples, - long failedChunks, - string lastError, - int streamDelayHintMs) - { - Active = active; - ReferenceAttached = referenceAttached; - CaptureSampleRate = captureSampleRate; - CaptureChannels = captureChannels; - ReferenceSampleRate = referenceSampleRate; - ReferenceChannels = referenceChannels; - CaptureChunks = captureChunks; - ReferenceChunks = referenceChunks; - DroppedCaptureSamples = droppedCaptureSamples; - DroppedReferenceSamples = droppedReferenceSamples; - FailedChunks = failedChunks; - LastError = lastError; - StreamDelayHintMs = streamDelayHintMs; - } - - public override string ToString() => - $"active={Active} reference={ReferenceAttached} " + - $"capture={CaptureChannels}ch@{CaptureSampleRate} chunks={CaptureChunks} dropped={DroppedCaptureSamples} | " + - $"reference={ReferenceChannels}ch@{ReferenceSampleRate} chunks={ReferenceChunks} dropped={DroppedReferenceSamples} | " + - $"failed={FailedChunks} lastError={LastError ?? "-"} delayHint={StreamDelayHintMs}ms"; - } -} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta deleted file mode 100644 index 3174751e..00000000 --- a/Runtime/Scripts/Audio/Processing/AudioProcessingStats.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 8515244a296964046ae7f54887a9aa1b -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index ca72566c..41fab2b1 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -25,7 +25,7 @@ namespace LiveKit /// echo of tick N's playout only reaches the microphone several ticks later. , /// and the maintenance coroutine run on the main thread and never touch the /// ring buffers; they raise flags the audio thread acts on. Nothing here logs from the audio - /// thread — diagnostics are counters, read via . + /// thread; the two warnings it can raise are posted to the main thread. /// /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple /// of 10 ms, so the chunking here is not optional, and rates whose 10 ms chunk is not a whole @@ -73,14 +73,11 @@ internal sealed class AudioProcessor : IDisposable private int _captureResetRequested; private int _referenceResetRequested; private int _unsupportedRateWarned; + private int _moduleFailureWarned; private int _maintenanceGeneration; - // Counters. - private long _captureChunks; - private long _referenceChunks; - private long _failedChunks; - private volatile string _lastError; - private volatile int _delayHintMs = -1; + // Main thread only. + private int _delayHintMs = -1; /// The FFI could not create the module. public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) @@ -207,7 +204,6 @@ public bool TryProcessCapture(float[] data, int channels, int sampleRate) var frame = new NativeArray(_captureChunkSamples, Allocator.Persistent); frame.CopyFrom(_captureChunk); ProcessCaptureChunk(frame, sampleRate, channels); - Interlocked.Increment(ref _captureChunks); _sink(frame, channels, sampleRate); } @@ -224,12 +220,12 @@ private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int c ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(frame); } var error = _apm.ProcessStream(ptr, frame.Length * sizeof(short), sampleRate, channels); - if (error != null) RecordFailure(error); + if (error != null) WarnModuleFailure(error); } catch (Exception e) { // The chunk goes out unprocessed rather than not at all. - RecordFailure(e.Message); + WarnModuleFailure(e.Message); } } @@ -260,13 +256,12 @@ private void OnPlayoutAudio(float[] data, int channels, int sampleRate) try { var error = _apm.ProcessReverseStream(_referenceChunkPin.AddrOfPinnedObject(), byteCount, sampleRate, channels); - if (error != null) RecordFailure(error); + if (error != null) WarnModuleFailure(error); } catch (Exception e) { - RecordFailure(e.Message); + WarnModuleFailure(e.Message); } - Interlocked.Increment(ref _referenceChunks); } } } @@ -300,19 +295,26 @@ private void ConfigureReference(int sampleRate, int channels, int incomingSample _referenceRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); } - private void RecordFailure(string error) + // The module rejected a chunk, which went out unprocessed. Warn once. + private void WarnModuleFailure(string error) { - _lastError = error; - Interlocked.Increment(ref _failedChunks); + if (Interlocked.Exchange(ref _moduleFailureWarned, 1) == 1) return; + + PostWarning("AudioProcessor: the audio processing module rejected a chunk, which was published " + + $"unprocessed: {error}"); } - // Logging is not allowed on the audio thread; hand the message to the main thread. private void WarnUnsupportedRate(int sampleRate) { if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; - var message = $"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + - "audio processing is bypassed and the capture is published unprocessed."; + PostWarning($"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + + "audio processing is bypassed and the capture is published unprocessed."); + } + + // Logging is not allowed on the audio thread; hand the message to the main thread. + private static void PostWarning(string message) + { var context = FfiClient.Instance._context; if (context != null) context.Post(static m => Utils.Warning(m), message); @@ -320,21 +322,6 @@ private void WarnUnsupportedRate(int sampleRate) Utils.Warning(message); } - public AudioProcessingStats GetStats() => new AudioProcessingStats( - active: _running && !_bypass && !_disposed, - referenceAttached: _echoCancellation && PlayoutReference.IsAttached, - captureSampleRate: _captureRate, - captureChannels: _captureChannels, - referenceSampleRate: _referenceRate, - referenceChannels: _referenceChannels, - captureChunks: Interlocked.Read(ref _captureChunks), - referenceChunks: Interlocked.Read(ref _referenceChunks), - droppedCaptureSamples: _captureRing?.OverflowSamples ?? 0, - droppedReferenceSamples: _referenceRing?.OverflowSamples ?? 0, - failedChunks: Interlocked.Read(ref _failedChunks), - lastError: _lastError, - streamDelayHintMs: _delayHintMs); - public void Dispose() { if (_disposed) return; diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index ee764916..74dcb5e0 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -56,9 +56,6 @@ private sealed class PendingAudioFrame /// created and the source fell back to unprocessed capture. /// public bool AudioProcessingEnabled => _processor != null; - - /// Counters from the audio processing stage; default when processing is off. - public AudioProcessingStats AudioProcessingStats => _processor?.GetStats() ?? default; private readonly int _debugId = Interlocked.Increment(ref nextDebugId); internal readonly uint _expectedSampleRate; internal readonly uint _expectedChannels; diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 29314dfa..88fa64bf 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -665,21 +665,6 @@ private IEnumerator PublishLocalMicrophoneUnity() Debug.Log(rtcSource.AudioProcessingEnabled ? "Microphone published via Unity Microphone API (audio processing active)" : "Microphone published via Unity Microphone API (no audio processing)"); - - if (rtcSource.AudioProcessingEnabled) - StartCoroutine(LogAudioProcessingStats(rtcSource)); - } - - // Periodic snapshot of the processing stage while the Unity microphone is published: confirms - // the playout reference is attached and that capture and reference chunks are flowing. - private IEnumerator LogAudioProcessingStats(RtcAudioSource source) - { - while (_microphoneActive && ReferenceEquals(_localRtcAudioSource, source)) - { - yield return new WaitForSeconds(5f); - if (!_microphoneActive || !ReferenceEquals(_localRtcAudioSource, source)) yield break; - Debug.Log($"Audio processing: {source.AudioProcessingStats}"); - } } private void UnpublishLocalMicrophone() diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs index 5e5f5c73..bc6145c0 100644 --- a/Tests/PlayMode/AudioProcessingTests.cs +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -41,8 +41,41 @@ public void AudioProcessingModule_AcceptsTenMillisecondChunks() } } + [Test] + public void AudioProcessor_ChunksCaptureIntoTenMilliseconds() + { + var rate = AudioSettings.outputSampleRate; + const int channels = 2; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var chunkSamples = AudioProcessingModule.FrameSizeFor(rate) * channels; + var counter = new ChunkCounter(chunkSamples); + using var processor = new AudioProcessor(AudioProcessingOptions.Default, counter.OnProcessed); + processor.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + Assert.IsTrue(processor.TryProcessCapture(block, channels, rate), "block was not processed"); + + var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); + Assert.That(counter.Chunks, Is.InRange(expectedChunks - 1, expectedChunks)); + Assert.AreEqual(0, counter.WrongSizedChunks, "a frame was not exactly one 10 ms chunk"); + + processor.Stop(); + } + + /// + /// The source-level path: options create the processor, and the capture the source reads goes + /// through it to the FFI. The Rust side takes the process down on a frame that is not 10 ms, + /// so getting through the pushes is the check; the exact chunking is covered above. + /// [UnityTest] - public IEnumerator RtcAudioSource_WithProcessing_CapturesInTenMillisecondChunks() + public IEnumerator RtcAudioSource_WithProcessing_PublishesPushedCapture() { using var source = new PushAudioSource(AudioProcessingOptions.Default); Assert.IsTrue(source.AudioProcessingEnabled, "module creation failed"); @@ -65,13 +98,6 @@ public IEnumerator RtcAudioSource_WithProcessing_CapturesInTenMillisecondChunks( // Let the capture callbacks return before disposing. yield return new WaitForSeconds(0.2f); - var stats = source.AudioProcessingStats; - var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); - Assert.IsTrue(stats.Active, stats.ToString()); - Assert.AreEqual(0, stats.FailedChunks, stats.LastError); - Assert.That(stats.CaptureChunks, Is.InRange(expectedChunks - 1, expectedChunks), stats.ToString()); - Assert.AreEqual(0, stats.DroppedCaptureSamples, stats.ToString()); - source.Stop(); } @@ -120,7 +146,6 @@ public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() far.PlayScheduled(startTime); near.PlayScheduled(startTime + 0.12); - AudioProcessingStats stats; float rawRms, processedRms; try { @@ -135,7 +160,6 @@ public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() yield return new WaitForSeconds(1.5f); (rawRms, processedRms) = meter.Window(); - stats = processor.GetStats(); } finally { @@ -146,12 +170,10 @@ public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() } Assert.Greater(rawRms, 0.01f, "near-end source produced no signal"); - Assert.AreEqual(0, stats.FailedChunks, stats.ToString()); - Assert.Greater(stats.ReferenceChunks, 0, stats.ToString()); var attenuationDb = 20f * Mathf.Log10(rawRms / Mathf.Max(processedRms, 1e-6f)); - Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB ({stats})"); - Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB; {stats}"); + Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB"); + Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB"); } private static AudioClip NoiseClip(int sampleRate, float seconds, int seed, float amplitude) @@ -177,6 +199,25 @@ public PushAudioSource(AudioProcessingOptions options) public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); } + // Counts the frames the processor hands out and checks each one is exactly one chunk. The + // sink owns and disposes the frames it is handed. + private sealed class ChunkCounter + { + private readonly int _chunkSamples; + + public ChunkCounter(int chunkSamples) => _chunkSamples = chunkSamples; + + public int Chunks { get; private set; } + public int WrongSizedChunks { get; private set; } + + public void OnProcessed(NativeArray frame, int channels, int sampleRate) + { + if (frame.Length != _chunkSamples) WrongSizedChunks++; + Chunks++; + frame.Dispose(); + } + } + // Accumulates energy of the raw near end and of the processed output. Both callbacks run on // the Unity audio thread; the sink owns and disposes the frames it is handed. private sealed class EchoMeter From 148d3e0a95217af5f9448720676c1336e919e477 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:33:54 +0200 Subject: [PATCH 06/11] Drop remoteAudioGain from the Meet sample Remote AudioSources play at Unity's default volume again. The 0.7 came over from the review package as their tuning; on-device testing at full gain was acceptable, and the sample should start with fewer knobs. The custom editor loses its Unity Audio section and shows the AEC3 note directly under the processing toggles when PlatformAudio is off. Co-Authored-By: Claude Fable 5.1 --- .../Meet/Assets/Editor/MeetManagerEditor.cs | 19 ++++--------------- Samples~/Meet/Assets/Runtime/MeetManager.cs | 17 +++++------------ Samples~/Meet/Assets/Scenes/MeetApp.unity | 1 - 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 0fb70078..f1321618 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -13,7 +13,6 @@ public class MeetManagerEditor : Editor private SerializedProperty noiseSuppression; private SerializedProperty autoGainControl; private SerializedProperty preferHardwareProcessing; - private SerializedProperty remoteAudioGain; private void OnEnable() { @@ -26,7 +25,6 @@ private void OnEnable() noiseSuppression = serializedObject.FindProperty("noiseSuppression"); autoGainControl = serializedObject.FindProperty("autoGainControl"); preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); - remoteAudioGain = serializedObject.FindProperty("remoteAudioGain"); } public override void OnInspectorGUI() @@ -69,20 +67,11 @@ public override void OnInspectorGUI() "different quality characteristics.")); } - EditorGUILayout.Space(); - EditorGUILayout.LabelField("Unity Audio (PlatformAudio off)", EditorStyles.boldLabel); - - // Gray out Unity audio options when PlatformAudio is enabled - using (new EditorGUI.DisabledGroupScope(platformAudioEnabled)) + if (!platformAudioEnabled) { - EditorGUILayout.HelpBox(platformAudioEnabled - ? "Unity audio options are only used when 'Use Platform Audio' is disabled." - : "Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is the mix on the " + - "AudioListener (a PlayoutReference component is attached automatically).", MessageType.Info); - - EditorGUILayout.PropertyField(remoteAudioGain, new GUIContent("Remote Audio Gain", - "Playback gain for every remote AudioSource. Below 1 keeps headroom so full-volume playout does not " + - "distort or overload the echo canceller. 0.7 is -3.1 dB.")); + EditorGUILayout.HelpBox("Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is " + + "the mix on the AudioListener (a PlayoutReference component is attached automatically).", + MessageType.Info); } serializedObject.ApplyModifiedProperties(); diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 88fa64bf..f545a6d0 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -47,13 +47,6 @@ public class MeetManager : MonoBehaviour [Tooltip("PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; - [Header("Unity Audio (PlatformAudio off)")] - [Tooltip("Playback gain for every remote AudioSource in Unity audio mode. Kept below 1 so a device " + - "at full speaker volume keeps headroom: full-scale playout distorts on Android and feeds " + - "the echo canceller more echo than it can remove. Linear amplitude, 0.7 is -3.1 dB.")] - [Range(0f, 1f)] - [SerializeField] private float remoteAudioGain = 0.7f; - private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; private Texture _placeholderTexture; @@ -209,10 +202,8 @@ private void OnToggleMicrophone() private void OnPublishData() { - Debug.Log($"Published Data"); - var bytes = System.Text.Encoding.Default.GetBytes("hello from unity!"); - _room.LocalParticipant.PublishData(bytes); - _room.LocalParticipant.SendText("Hello from Unity, Max", "Chat"); + echoCancellation = !echoCancellation; + Debug.Log($"use AEC is {echoCancellation}"); } #endregion @@ -382,7 +373,6 @@ private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) audioObject.transform.SetParent(_audioTrackParent); var source = audioObject.AddComponent(); - source.volume = remoteAudioGain; var audiostream = new AudioStream(audioTrack, source); _audioStreams.Add(sid, audiostream); @@ -634,6 +624,9 @@ private IEnumerator PublishLocalMicrophoneUnity() NoiseSuppression = noiseSuppression, AutoGainControl = autoGainControl }; + + Debug.Log($"Created mic with echo cancellation {echoCancellation}"); + var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 3e03e2d1..dd50f588 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -907,7 +907,6 @@ MonoBehaviour: noiseSuppression: 1 autoGainControl: 1 preferHardwareProcessing: 1 - remoteAudioGain: 0.7 --- !u!114 &1478206705 MonoBehaviour: m_ObjectHideFlags: 0 From d77a8e699dfa7f785aa1cc2850142dc7778cde96 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:21:01 +0200 Subject: [PATCH 07/11] Some renaming in PlayoutReference --- .../Audio/Processing/PlayoutReference.cs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs index 2693b4a7..b0e2f8b1 100644 --- a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs @@ -6,9 +6,14 @@ namespace LiveKit { /// - /// Taps the final mix Unity sends to the audio device and feeds it to the echo canceller as - /// the far-end reference. Lives on the GameObject of the active . + /// The the final mix Unity sends to the audio hardware device and + /// feeds it to the echo canceller as the far-end reference. + /// + /// Attaches itself to the GameObject of the active , which is the virtual + /// microphone in the scene, usually sitting on the virtual camera object, capturing the virtual sound + /// and sending it to the audio output hardware. /// /// + /// /// /// An created with /// attaches this component to the active listener when it starts and re-attaches it after @@ -23,7 +28,6 @@ namespace LiveKit /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so /// the sample rate and listener state are cached on the main thread. /// - [AddComponentMenu("LiveKit/Playout Reference")] public sealed class PlayoutReference : MonoBehaviour { internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); @@ -34,15 +38,17 @@ public sealed class PlayoutReference : MonoBehaviour /// internal static event PlayoutAudioDelegate AudioRead; - private static PlayoutReference _active; + // Singleton pattern instance + private static PlayoutReference _instance; private static int _consumers; + // The AudioListener we are attached to private AudioListener _listener; private volatile int _sampleRate; private volatile bool _deliver; /// Whether a reference on an enabled listener is delivering audio. - internal static bool IsAttached => _active != null && _active._deliver; + internal static bool IsAttached => _instance != null && _instance._deliver; /// Main thread. Registers a consumer and attaches to the listener if possible. internal static void Acquire() @@ -65,15 +71,15 @@ internal static void Release() internal static void EnsureAttached() { if (_consumers == 0) return; - if (_active != null && _active.isActiveAndEnabled && - _active._listener != null && _active._listener.isActiveAndEnabled) + if (_instance != null && _instance.isActiveAndEnabled && + _instance._listener != null && _instance._listener.isActiveAndEnabled) return; var listener = FindActiveListener(); if (listener == null) return; var existing = listener.GetComponent(); - _active = existing != null ? existing : listener.gameObject.AddComponent(); + _instance = existing != null ? existing : listener.gameObject.AddComponent(); } private static AudioListener FindActiveListener() @@ -94,14 +100,14 @@ private void OnEnable() RefreshDeliveryState(); AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; - if (_active == null) _active = this; + if (_instance == null) _instance = this; } private void OnDisable() { AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; _deliver = false; - if (_active == this) _active = null; + if (_instance == this) _instance = null; } private void Update() @@ -136,7 +142,7 @@ private static IEnumerator Reattach(GameObject host) yield return null; if (host == null || _consumers == 0) yield break; if (host.GetComponent() == null) - _active = host.AddComponent(); + _instance = host.AddComponent(); } // Unity audio thread. From 7a9b62181f112d404fbb1149c0ef6a7f59c34394 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:00:42 +0200 Subject: [PATCH 08/11] Address review findings on the Unity-audio AEC path - Meet scene: point the token config back at the tracked asset instead of a gitignored local config - Meet sample: restore the Publish Data button and drop a duplicate log; stop calling PlatformAudio the default in the class comment - RtcAudioSource: keep feeding the processing stage while muted so AEC3 sees the near end next to its reference; the processed output is dropped instead - AudioProcessor: only run the maintenance loop for echo cancellation and only when a coroutine host exists, since the loop never finishes and would otherwise be drained synchronously; use unscaled time; check the sample rate per block instead of latching a bypass; Dispose(bool) keeps the finalizer path off PlayoutReference state - PlayoutReference: fix the doc header, add DisallowMultipleComponent Co-Authored-By: Claude Fable 5.1 --- .../Audio/Processing/AudioProcessor.cs | 39 ++++++++++++------- .../Audio/Processing/PlayoutReference.cs | 14 +++---- Runtime/Scripts/Audio/RtcAudioSource.cs | 12 ++++-- .../Threading/MonoBehaviourContext.cs | 6 +++ Samples~/Meet/Assets/Runtime/MeetManager.cs | 13 ++++--- Samples~/Meet/Assets/Scenes/MeetApp.unity | 2 +- 6 files changed, 56 insertions(+), 30 deletions(-) diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index 41fab2b1..254b41f7 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -28,8 +28,8 @@ namespace LiveKit /// thread; the two warnings it can raise are posted to the main thread. /// /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple - /// of 10 ms, so the chunking here is not optional, and rates whose 10 ms chunk is not a whole - /// number of samples are bypassed entirely. + /// of 10 ms, so the chunking here is not optional, and blocks at a rate whose 10 ms chunk is + /// not a whole number of samples are bypassed. /// internal sealed class AudioProcessor : IDisposable { @@ -68,7 +68,6 @@ internal sealed class AudioProcessor : IDisposable // Cross-thread flags. private volatile bool _running; - private volatile bool _bypass; private volatile bool _disposed; private int _captureResetRequested; private int _referenceResetRequested; @@ -108,7 +107,10 @@ public void Start() } SeedDelayHint(); - MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); + // Only echo cancellation has periodic upkeep. The loop never finishes on its own, so it + // is not handed to a host that would drain it synchronously. + if (_echoCancellation && MonoBehaviourContext.CanRunCoroutines) + MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); } /// Main thread. @@ -134,15 +136,16 @@ public void RequestReset() Interlocked.Exchange(ref _referenceResetRequested, 1); } - // Periodic main-thread upkeep: re-attach the reference after scene or device changes and - // refresh the delay hint (iOS reports zero session latency until the session is active). + // Periodic main-thread upkeep for echo cancellation: re-attach the reference after scene or + // device changes and refresh the delay hint (iOS reports zero session latency until the + // session is active). Unscaled time, so a paused game keeps its reference. private IEnumerator Maintenance(int generation) { while (_running && !_disposed && generation == _maintenanceGeneration) { - if (_echoCancellation) PlayoutReference.EnsureAttached(); + PlayoutReference.EnsureAttached(); SeedDelayHint(); - yield return new WaitForSeconds(MaintenanceIntervalSeconds); + yield return new WaitForSecondsRealtime(MaintenanceIntervalSeconds); } } @@ -179,12 +182,13 @@ private void SeedDelayHint() /// public bool TryProcessCapture(float[] data, int channels, int sampleRate) { - if (_disposed || !_running || _bypass) return false; + if (_disposed || !_running) return false; if (data == null || data.Length == 0 || channels <= 0 || sampleRate <= 0) return false; + // Checked per block rather than latched: Unity's output rate can change with the + // device, and a rate that becomes supported later is processed again. if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) { - _bypass = true; WarnUnsupportedRate(sampleRate); return false; } @@ -232,7 +236,7 @@ private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int c // Unity audio thread, from PlayoutReference. Must not modify data. private void OnPlayoutAudio(float[] data, int channels, int sampleRate) { - if (_disposed || !_running || _bypass) return; + if (_disposed || !_running) return; if (data == null || data.Length == 0 || channels <= 0) return; if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) return; @@ -322,11 +326,20 @@ private static void PostWarning(string message) Utils.Warning(message); } - public void Dispose() + public void Dispose() => Dispose(true); + + /// + /// is false on the owner's finalizer path: only the native + /// module and the pinned chunk are released there. The + /// registration is main-thread state and is only touched by . + /// + internal void Dispose(bool disposing) { if (_disposed) return; - Stop(); + if (disposing) Stop(); + else _running = false; + lock (_referenceLock) { _disposed = true; diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs index b0e2f8b1..dfdd78f1 100644 --- a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs @@ -6,14 +6,13 @@ namespace LiveKit { /// - /// The the final mix Unity sends to the audio hardware device and - /// feeds it to the echo canceller as the far-end reference. - /// - /// Attaches itself to the GameObject of the active , which is the virtual - /// microphone in the scene, usually sitting on the virtual camera object, capturing the virtual sound - /// and sending it to the audio output hardware. /// + /// Taps the final mix Unity sends to the audio output device and feeds it to the echo + /// canceller as the far-end reference. + /// + /// Attaches itself to the GameObject of the active , the virtual + /// microphone in the scene, usually on the camera, which hears the scene and sends the result + /// to the audio output hardware. /// - /// /// /// An created with /// attaches this component to the active listener when it starts and re-attaches it after @@ -28,6 +27,7 @@ namespace LiveKit /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so /// the sample rate and listener state are cached on the main thread. /// + [DisallowMultipleComponent] public sealed class PlayoutReference : MonoBehaviour { internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 74dcb5e0..da5faa4a 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -207,8 +207,11 @@ public virtual void Stop() private void OnAudioRead(float[] data, int channels, int sampleRate) { - if (_muted) return; if (_disposed) return; + // A muted block still runs through the processing stage so the echo canceller keeps + // seeing the near end next to its reference; SendProcessedFrame drops the output. + // Without a processing stage there is nothing to keep warm. + if (_muted && _processor == null) return; var readIndex = Interlocked.Increment(ref _audioReadCount); if (channels <= 0) @@ -233,6 +236,8 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_processor != null && _processor.TryProcessCapture(data, channels, sampleRate)) return; + if (_muted) return; + // Each captured frame gets its own backing buffer so the native encoder can safely // consume it asynchronously after request.Send() returns. var frameData = new NativeArray(data.Length, Allocator.Persistent); @@ -242,7 +247,8 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) SendFrame(frameData, channels, sampleRate); } - // Audio thread, from the processing stage. Owns the frame from here on. + // Audio thread, from the processing stage. Owns the frame from here on. Muted output is + // dropped here, after the module has seen the block. private void SendProcessedFrame(NativeArray frame, int channels, int sampleRate) { if (_disposed || _muted) @@ -385,7 +391,7 @@ protected virtual void Dispose(bool disposing) } _pendingFrameData.Clear(); } - _processor?.Dispose(); + _processor?.Dispose(disposing); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); diff --git a/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs b/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs index 487122d1..75a0c97c 100644 --- a/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs +++ b/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs @@ -13,6 +13,12 @@ internal class MonoBehaviourContext : MonoBehaviour private static MonoBehaviourContext _instance; private const string OBJECT_NAME = "LiveKitSDK"; + /// + /// Whether a host exists to run coroutines over time. When false, + /// drains the coroutine synchronously, which is only safe for one that finishes on its own. + /// + internal static bool CanRunCoroutines => _instance != null; + [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.BeforeSceneLoad)] private static void Init() { diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index f545a6d0..5000de12 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -9,8 +9,9 @@ /// /// Manages a LiveKit room connection with local/remote audio and video tracks. /// -/// Supports two audio modes: -/// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic +/// Supports two audio modes, selected with usePlatformAudio (the Meet scene ships with +/// Unity Audio selected): +/// - PlatformAudio: Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. /// Gives more control over audio processing. The same AEC/NS/AGC toggles apply: the SDK runs @@ -202,8 +203,10 @@ private void OnToggleMicrophone() private void OnPublishData() { - echoCancellation = !echoCancellation; - Debug.Log($"use AEC is {echoCancellation}"); + Debug.Log($"Published Data"); + var bytes = System.Text.Encoding.Default.GetBytes("hello from unity!"); + _room.LocalParticipant.PublishData(bytes); + _room.LocalParticipant.SendText("Hello from Unity, Max", "Chat"); } #endregion @@ -625,8 +628,6 @@ private IEnumerator PublishLocalMicrophoneUnity() AutoGainControl = autoGainControl }; - Debug.Log($"Created mic with echo cancellation {echoCancellation}"); - var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index dd50f588..a581e401 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -919,7 +919,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a498c208deeab40c39b4ba609d7d222c, type: 3} m_Name: m_EditorClassIdentifier: - _config: {fileID: 11400000, guid: aae6c4b1158ca4c929c5d84962c95f91, type: 2} + _config: {fileID: 11400000, guid: 1a1b8efb5101449b280f574f853b7459, type: 2} --- !u!1 &1651282853 GameObject: m_ObjectHideFlags: 0 From 0a23b8e41c3eb0d887efcedaeb72266b75fae4aa Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:34:35 +0200 Subject: [PATCH 09/11] Add guard with fallback when Unity audio system is disabled in settings --- Runtime/Scripts/Audio/RtcAudioSource.cs | 26 +++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index da5faa4a..65bdc7a7 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -146,20 +146,34 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui } } + // Format used when Unity reports no usable output configuration. Matches the FFI defaults. + private const uint FallbackSampleRate = 48000; + private const uint FallbackChannels = 1; + // Reads Unity's actual output audio configuration. The capture path delivers buffers at the // DSP output rate/channel count (see AudioProbe), so this is the format the native source - // must match. Falls back to the platform defaults when Unity cannot report a configuration - // (e.g. batch mode without an audio device). + // must match. When the Unity audio system is disabled (Project Settings > Audio > Disable + // Unity Audio, dedicated servers) Unity reports a 0 Hz rate and the Raw speaker mode. The + // native source must not be created with that format: its 10 ms silence timer divides by + // the channel count once the track is published. Fall back to the FFI defaults and warn; + // capture through Unity audio cannot work in that state, but the process stays alive. private (uint sampleRate, uint channels) ResolveDeviceFormat() { var config = UnityEngine.AudioSettings.GetConfiguration(); - var sampleRate = (uint)config.sampleRate; - var configuredChannels = SpeakerModeChannels(config.speakerMode); - var channels = configuredChannels; + var sampleRate = config.sampleRate; + var channels = SpeakerModeChannels(config.speakerMode); + + if (sampleRate <= 0 || channels == 0) + { + Utils.Warning($"{DebugTag} Unity reports no usable output format (sampleRate={sampleRate}, " + + $"speakerMode={config.speakerMode}); the Unity audio system is probably disabled. " + + $"Falling back to {FallbackSampleRate} Hz, {FallbackChannels} channel(s)."); + return (FallbackSampleRate, FallbackChannels); + } Utils.Info($"Configured native audio source with sampleRate {sampleRate} and channels {channels}"); - return (sampleRate, channels); + return ((uint)sampleRate, channels); } private static uint SpeakerModeChannels(UnityEngine.AudioSpeakerMode mode) From 6182d551bb1d9a6a2f3c6d404d16430d0241812f Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:58:42 +0200 Subject: [PATCH 10/11] Refactor PlayoutReference to simplify the consumer logic --- .../Audio/Processing/AudioProcessor.cs | 10 +--- .../Audio/Processing/PlayoutReference.cs | 56 +++++++++++-------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index 254b41f7..1adfaacf 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -101,10 +101,7 @@ public void Start() RequestReset(); if (_echoCancellation) - { - PlayoutReference.AudioRead += OnPlayoutAudio; - PlayoutReference.Acquire(); - } + PlayoutReference.Acquire(OnPlayoutAudio); SeedDelayHint(); // Only echo cancellation has periodic upkeep. The loop never finishes on its own, so it @@ -120,10 +117,7 @@ public void Stop() _running = false; if (_echoCancellation) - { - PlayoutReference.AudioRead -= OnPlayoutAudio; - PlayoutReference.Release(); - } + PlayoutReference.Release(OnPlayoutAudio); } /// diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs index dfdd78f1..fd45d081 100644 --- a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs @@ -25,42 +25,51 @@ namespace LiveKit /// local microphone never appears in the mix. /// /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so - /// the sample rate and listener state are cached on the main thread. + /// the sample rate, listener state and consumer count are cached on the main thread. /// [DisallowMultipleComponent] public sealed class PlayoutReference : MonoBehaviour { internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); - /// - /// Raised on the Unity audio thread with the final mix. Subscribers must not modify the - /// buffer: it is on its way to the speaker. - /// - internal static event PlayoutAudioDelegate AudioRead; + // Raised on the Unity audio thread with the final mix. Consumers must not modify the + // buffer: it is on its way to the speaker. The invocation list is the consumer list, so + // a consumer cannot exist without holding the reference and vice versa. + private static event PlayoutAudioDelegate AudioRead; + private static bool HasConsumers => AudioRead != null; // Singleton pattern instance private static PlayoutReference _instance; - private static int _consumers; // The AudioListener we are attached to - private AudioListener _listener; + private AudioListener _sceneAudioListener; private volatile int _sampleRate; private volatile bool _deliver; - /// Whether a reference on an enabled listener is delivering audio. + /// Whether a reference on an enabled listener is delivering audio to a consumer. internal static bool IsAttached => _instance != null && _instance._deliver; - /// Main thread. Registers a consumer and attaches to the listener if possible. - internal static void Acquire() + /// + /// Main thread. Registers to receive the final mix and attaches + /// to the listener if possible. Acquiring the same consumer twice delivers to it twice + /// until it is released twice. + /// + internal static void Acquire(PlayoutAudioDelegate consumer) { - _consumers++; + AudioRead += consumer; EnsureAttached(); + if (_instance != null) _instance.RefreshDeliveryState(); } - /// Main thread. The component stays on the listener; it is inert without consumers. - internal static void Release() + /// + /// Main thread. Removes . The component stays on the listener; + /// once the last consumer is gone it stops delivering. Releasing a consumer that was not + /// acquired is a no-op. + /// + internal static void Release(PlayoutAudioDelegate consumer) { - if (_consumers > 0) _consumers--; + AudioRead -= consumer; + if (_instance != null) _instance.RefreshDeliveryState(); } /// @@ -70,9 +79,9 @@ internal static void Release() /// internal static void EnsureAttached() { - if (_consumers == 0) return; + if (!HasConsumers) return; if (_instance != null && _instance.isActiveAndEnabled && - _instance._listener != null && _instance._listener.isActiveAndEnabled) + _instance._sceneAudioListener != null && _instance._sceneAudioListener.isActiveAndEnabled) return; var listener = FindActiveListener(); @@ -94,8 +103,8 @@ private static AudioListener FindActiveListener() private void OnEnable() { - _listener = GetComponent(); - if (_listener == null) + _sceneAudioListener = GetComponent(); + if (_sceneAudioListener == null) Utils.Warning("PlayoutReference must be on the AudioListener's GameObject; it will not deliver a reference from here."); RefreshDeliveryState(); @@ -115,11 +124,12 @@ private void Update() RefreshDeliveryState(); } - // Listener state and the output rate are Unity APIs; sample them here for the audio thread. + // Listener state and the output rate are Unity APIs and the consumer list is mutated on the + // main thread; fold them into one flag so the audio thread reads a single volatile. private void RefreshDeliveryState() { _sampleRate = AudioSettings.outputSampleRate; - _deliver = _listener != null && _listener.isActiveAndEnabled; + _deliver = HasConsumers && _sceneAudioListener != null && _sceneAudioListener.isActiveAndEnabled; } // Unity rebuilds the DSP graph on a device change (or AudioSettings.Reset), which can leave @@ -129,7 +139,7 @@ private void RefreshDeliveryState() private void OnAudioConfigurationChanged(bool deviceWasChanged) { RefreshDeliveryState(); - if (_consumers == 0) return; + if (!HasConsumers) return; var host = gameObject; Destroy(this); @@ -140,7 +150,7 @@ private static IEnumerator Reattach(GameObject host) { // Let the deferred Destroy apply before adding the replacement. yield return null; - if (host == null || _consumers == 0) yield break; + if (host == null || !HasConsumers) yield break; if (host.GetComponent() == null) _instance = host.AddComponent(); } From 243852569b6a6b6e50403689e5a7dd06a74392d0 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:22:13 +0200 Subject: [PATCH 11/11] Exchanging new native array allocation for every frame for a reusable short array --- .../Audio/Processing/AudioProcessor.cs | 22 +- Runtime/Scripts/Audio/RtcAudioSource.cs | 200 +++++------------- Tests/PlayMode/AudioProcessingTests.cs | 12 +- Tests/PlayMode/Utils/SineWaveAudioSource.cs | 8 + 4 files changed, 69 insertions(+), 173 deletions(-) diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs index 1adfaacf..f77a6f38 100644 --- a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -5,8 +5,6 @@ using LiveKit.Internal; using LiveKit.Internal.FFI; using LiveKit.Internal.Threading; -using Unity.Collections; -using Unity.Collections.LowLevel.Unsafe; using UnityEngine; namespace LiveKit @@ -33,8 +31,8 @@ namespace LiveKit /// internal sealed class AudioProcessor : IDisposable { - /// Receives one processed 10 ms chunk and takes ownership of the array. - internal delegate void ProcessedFrameSink(NativeArray frame, int channels, int sampleRate); + /// Receives one processed 10 ms chunk. The frame is valid only during the call. + internal delegate void ProcessedFrameSink(ReadOnlySpan frame, int channels, int sampleRate); // Ring capacity in chunks. Bounds the latency added when a DSP block is not a multiple of 10 ms. private const int BufferedChunks = 8; @@ -45,7 +43,7 @@ internal sealed class AudioProcessor : IDisposable private readonly bool _echoCancellation; // Guards the pinned reference chunk against Dispose racing an in-flight callback. The - // capture side needs no lock: each chunk lives in its own NativeArray owned by the sink. + // capture side needs no lock: its chunk is pinned only for the duration of each call. private readonly object _referenceLock = new object(); // Capture side. Audio thread only. @@ -199,25 +197,23 @@ public bool TryProcessCapture(float[] data, int channels, int sampleRate) while (_captureRing.TryDrain(_captureChunk, _captureChunkSamples)) { - var frame = new NativeArray(_captureChunkSamples, Allocator.Persistent); - frame.CopyFrom(_captureChunk); - ProcessCaptureChunk(frame, sampleRate, channels); - _sink(frame, channels, sampleRate); + ProcessCaptureChunk(_captureChunk, sampleRate, channels); + _sink(_captureChunk, channels, sampleRate); } return true; } - private void ProcessCaptureChunk(NativeArray frame, int sampleRate, int channels) + private void ProcessCaptureChunk(short[] chunk, int sampleRate, int channels) { try { - IntPtr ptr; + string error; unsafe { - ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(frame); + fixed (short* ptr = chunk) + error = _apm.ProcessStream((IntPtr)ptr, chunk.Length * sizeof(short), sampleRate, channels); } - var error = _apm.ProcessStream(ptr, frame.Length * sizeof(short), sampleRate, channels); if (error != null) WarnModuleFailure(error); } catch (Exception e) diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 65bdc7a7..23891eab 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -1,12 +1,8 @@ using System; using System.Collections; -using System.Collections.Generic; using LiveKit.Proto; using LiveKit.Internal; using LiveKit.Internal.FFI.Requests; -using Unity.Collections; -using Unity.Collections.LowLevel.Unsafe; -using System.Diagnostics; using System.Threading; using LiveKit.Internal.FFI; @@ -26,16 +22,6 @@ public enum RtcAudioSourceType /// public abstract class RtcAudioSource : IRtcSource, IDisposable { - private sealed class PendingAudioFrame - { - public NativeArray FrameData; - public int FrameIndex; - public int SampleRate; - public int Channels; - public int SampleCount; - public long StartedTimestamp; - } - private static int nextDebugId = 0; /// @@ -43,7 +29,8 @@ private sealed class PendingAudioFrame /// Provides the audio data, channel count, and sample rate. /// /// - /// This event is not guaranteed to be called on the main thread. + /// This event is not guaranteed to be called on the main thread. It must not be invoked + /// concurrently: the source converts each block into one reusable buffer. /// public abstract event Action AudioRead; @@ -64,15 +51,15 @@ private sealed class PendingAudioFrame protected AudioSourceInfo _info; private readonly AudioProcessor _processor; - // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM - // pointer after request.Send() returns and encode it later on another queue. Because of - // that, a single reusable NativeArray is unsafe here; the next AudioRead callback can - // overwrite it while Opus/WebRTC is still consuming the previous frame. - // - // Keep one NativeArray per in-flight request and release it only after the matching - // CaptureAudioFrame callback completes or is canceled. - private readonly Dictionary _pendingFrameData = new(); - private readonly object _pendingFrameDataLock = new object(); + // CaptureAudioFrame copies the PCM into its own buffer on the calling thread before the + // request returns (livekit-ffi capture_frame, rust-sdks #289), so the pointer only has to + // stay valid for the duration of the synchronous Send(). One reusable buffer, pinned around + // the call, is enough. Audio thread only; see the AudioRead contract. + private short[] _captureBuffer = Array.Empty(); + + // Cached so a capture registers its callback without allocating per frame. + private readonly Action _onCaptureCallback; + private readonly Action _onCaptureCanceled; private volatile bool _muted = false; public override bool Muted => _muted; @@ -105,6 +92,8 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels, AudioProcessingOptions? processing) { _sourceType = audioSourceType; + _onCaptureCallback = OnCaptureCallback; + _onCaptureCanceled = OnCaptureCanceled; if (sampleRate > 0 && channels > 0) { @@ -212,11 +201,7 @@ public virtual void Stop() AudioRead -= OnAudioRead; _processor?.Stop(); _started = false; - var pendingCount = PendingFrameCount(); - if (pendingCount > 0) - Utils.Warning($"{DebugTag} stop requested with {pendingCount} pending capture callbacks"); - else - Utils.Debug($"{DebugTag} stop"); + Utils.Debug($"{DebugTag} stop"); } private void OnAudioRead(float[] data, int channels, int sampleRate) @@ -252,114 +237,66 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_muted) return; - // Each captured frame gets its own backing buffer so the native encoder can safely - // consume it asynchronously after request.Send() returns. - var frameData = new NativeArray(data.Length, Allocator.Persistent); + if (_captureBuffer.Length < data.Length) + _captureBuffer = new short[data.Length]; for (int i = 0; i < data.Length; i++) - frameData[i] = PcmConvert.FloatToS16(data[i]); + _captureBuffer[i] = PcmConvert.FloatToS16(data[i]); - SendFrame(frameData, channels, sampleRate); + SendFrame(new ReadOnlySpan(_captureBuffer, 0, data.Length), channels, sampleRate); } - // Audio thread, from the processing stage. Owns the frame from here on. Muted output is - // dropped here, after the module has seen the block. - private void SendProcessedFrame(NativeArray frame, int channels, int sampleRate) + // Audio thread, from the processing stage. Muted output is dropped here, after the module + // has seen the block. + private void SendProcessedFrame(ReadOnlySpan frame, int channels, int sampleRate) { - if (_disposed || _muted) - { - frame.Dispose(); - return; - } + if (_disposed || _muted) return; SendFrame(frame, channels, sampleRate); } - // Hands one int16 frame to the native source. Takes ownership of frameData: it is released - // when the CaptureAudioFrame callback completes, is canceled, or the send fails. - private void SendFrame(NativeArray frameData, int channels, int sampleRate) + // Hands one int16 frame to the native source. The frame is borrowed for the duration of + // the call only; the native side has its own copy when Send() returns. + private unsafe void SendFrame(ReadOnlySpan frame, int channels, int sampleRate) { var frameIndex = Interlocked.Increment(ref _sentFrameCount); - var pendingBeforeSend = PendingFrameCount(); - if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) + if (frameIndex <= 3 || frameIndex % 100 == 0) { - Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frameData.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frame.Length} channels={channels} sampleRate={sampleRate} thread={Thread.CurrentThread.ManagedThreadId}"); } - // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); using var audioFrameBufferInfo = request.TempResource(); var pushFrame = request.request; pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); pushFrame.Buffer = audioFrameBufferInfo; - unsafe - { - pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility - .GetUnsafePtr(frameData); - } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; - pushFrame.Buffer.SamplesPerChannel = (uint)frameData.Length / (uint)channels; - - // Wait for async callback, log an error if the capture fails. The callback's AsyncId - // echoes the RequestAsyncId that Unity wrote onto the request. - var requestAsyncId = request.RequestAsyncId; - var pendingFrame = new PendingAudioFrame - { - FrameData = frameData, - FrameIndex = frameIndex, - SampleRate = sampleRate, - Channels = channels, - SampleCount = frameData.Length, - StartedTimestamp = Stopwatch.GetTimestamp(), - }; - lock (_pendingFrameDataLock) - { - _pendingFrameData[requestAsyncId] = pendingFrame; - } + pushFrame.Buffer.SamplesPerChannel = (uint)(frame.Length / channels); - void Callback(CaptureAudioFrameCallback callback) - { - if (callback.AsyncId != requestAsyncId) return; - var completedFrame = ReleasePendingFrameData(requestAsyncId); - if (completedFrame != null) - { - var elapsedMs = ElapsedMilliseconds(completedFrame.StartedTimestamp); - if (callback.HasError) - { - Utils.Error($"{DebugTag} capture callback failed asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()} error={callback.Error}"); - } - else if (completedFrame.FrameIndex <= 3 || completedFrame.FrameIndex % 100 == 0 || elapsedMs > 100) - { - Utils.Debug($"{DebugTag} capture callback asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); - } - } - if (callback.HasError) - Utils.Error($"{DebugTag} audio capture failed: {callback.Error}"); - } - void OnCanceled() - { - var canceledFrame = ReleasePendingFrameData(requestAsyncId); - if (canceledFrame != null) - { - var elapsedMs = ElapsedMilliseconds(canceledFrame.StartedTimestamp); - Utils.Warning($"{DebugTag} capture callback canceled asyncId={requestAsyncId} frame={canceledFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); - } - } + // The callback only reports the outcome. It is registered before Send() so Rust cannot + // complete a request Unity has nowhere to store; Send() cancels it if the call throws. + FfiClient.Instance.RegisterPendingCallback(request.RequestAsyncId, static e => e.CaptureAudioFrame, _onCaptureCallback, _onCaptureCanceled); - FfiClient.Instance.RegisterPendingCallback(requestAsyncId, static e => e.CaptureAudioFrame, Callback, OnCanceled); - try + fixed (short* pcm = frame) { + pushFrame.Buffer.DataPtr = (ulong)pcm; using var response = request.Send(); } - catch - { - var failedFrame = ReleasePendingFrameData(requestAsyncId); - if (failedFrame != null) - { - Utils.Error($"{DebugTag} request send failed asyncId={requestAsyncId} frame={failedFrame.FrameIndex} pendingAfter={PendingFrameCount()}"); - } - throw; - } + } + + // Main thread, posted by the FFI client. + private void OnCaptureCallback(CaptureAudioFrameCallback callback) + { + if (callback.HasError) + Utils.Error($"{DebugTag} audio capture failed asyncId={callback.AsyncId}: {callback.Error}"); + } + + // The FFI client dropped the pending callback (dispose, resume window, or a failed send). + // Debug level: at quit the client sweeps every frame whose callback Rust never sent, one + // line per frame, and a failed send is already logged as an error by the client. + private void OnCaptureCanceled() + { + Utils.Debug($"{DebugTag} capture callback canceled"); } /// @@ -392,48 +329,12 @@ protected virtual void Dispose(bool disposing) if (disposing) Stop(); - var pendingCount = PendingFrameCount(); - if (pendingCount > 0) - Utils.Warning($"{DebugTag} dispose(disposing={disposing}) with {pendingCount} pending capture callbacks"); - - lock (_pendingFrameDataLock) - { - foreach (var pendingFrame in _pendingFrameData.Values) - { - if (pendingFrame.FrameData.IsCreated) - pendingFrame.FrameData.Dispose(); - } - _pendingFrameData.Clear(); - } _processor?.Dispose(disposing); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); } - private PendingAudioFrame ReleasePendingFrameData(ulong requestAsyncId) - { - PendingAudioFrame pendingFrame = null; - lock (_pendingFrameDataLock) - { - if (_pendingFrameData.TryGetValue(requestAsyncId, out pendingFrame)) - _pendingFrameData.Remove(requestAsyncId); - } - - if (pendingFrame != null && pendingFrame.FrameData.IsCreated) - pendingFrame.FrameData.Dispose(); - - return pendingFrame; - } - - private int PendingFrameCount() - { - lock (_pendingFrameDataLock) - { - return _pendingFrameData.Count; - } - } - ~RtcAudioSource() { Dispose(false); @@ -449,11 +350,6 @@ public IEnumerator PrepareAndStart() yield break; } - private static double ElapsedMilliseconds(long startedTimestamp) - { - return (Stopwatch.GetTimestamp() - startedTimestamp) * 1000.0 / Stopwatch.Frequency; - } - private string DebugTag => $"RtcAudioSource#{_debugId}"; } } diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs index bc6145c0..b5d5266a 100644 --- a/Tests/PlayMode/AudioProcessingTests.cs +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -2,7 +2,6 @@ using System.Collections; using System.Runtime.InteropServices; using NUnit.Framework; -using Unity.Collections; using UnityEngine; using UnityEngine.TestTools; @@ -199,8 +198,7 @@ public PushAudioSource(AudioProcessingOptions options) public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); } - // Counts the frames the processor hands out and checks each one is exactly one chunk. The - // sink owns and disposes the frames it is handed. + // Counts the frames the processor hands out and checks each one is exactly one chunk. private sealed class ChunkCounter { private readonly int _chunkSamples; @@ -210,16 +208,15 @@ private sealed class ChunkCounter public int Chunks { get; private set; } public int WrongSizedChunks { get; private set; } - public void OnProcessed(NativeArray frame, int channels, int sampleRate) + public void OnProcessed(ReadOnlySpan frame, int channels, int sampleRate) { if (frame.Length != _chunkSamples) WrongSizedChunks++; Chunks++; - frame.Dispose(); } } // Accumulates energy of the raw near end and of the processed output. Both callbacks run on - // the Unity audio thread; the sink owns and disposes the frames it is handed. + // the Unity audio thread. private sealed class EchoMeter { private readonly object _lock = new object(); @@ -243,7 +240,7 @@ public void OnRaw(float[] data) } } - public void OnProcessed(NativeArray frame, int channels, int sampleRate) + public void OnProcessed(ReadOnlySpan frame, int channels, int sampleRate) { double sum = 0; var length = frame.Length; @@ -252,7 +249,6 @@ public void OnProcessed(NativeArray frame, int channels, int sampleRate) var v = frame[i] / 32768.0; sum += v * v; } - frame.Dispose(); lock (_lock) { _processedSum += sum; diff --git a/Tests/PlayMode/Utils/SineWaveAudioSource.cs b/Tests/PlayMode/Utils/SineWaveAudioSource.cs index 2337615b..7bfa0448 100644 --- a/Tests/PlayMode/Utils/SineWaveAudioSource.cs +++ b/Tests/PlayMode/Utils/SineWaveAudioSource.cs @@ -25,6 +25,7 @@ public sealed class SineWaveAudioSource : RtcAudioSource private double _phase; private bool _running; private bool _disposed; + private int _pumping; public SineWaveAudioSource( int channels = 2, @@ -60,6 +61,9 @@ public override void Stop() private void PumpFrame(object _) { if (!_running || _disposed) return; + // Timer callbacks can overlap on the thread pool; AudioRead must not be invoked + // concurrently, so a tick that finds the previous one still running is skipped. + if (Interlocked.Exchange(ref _pumping, 1) == 1) return; try { var buffer = new float[_samplesPerFrame * _channels]; @@ -77,6 +81,10 @@ private void PumpFrame(object _) { // Timer fires independently of FFI lifecycle; swallow errors during teardown. } + finally + { + Interlocked.Exchange(ref _pumping, 0); + } } protected override void Dispose(bool disposing)