diff --git a/README.md b/README.md index 284dee4c..5b9114ce 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,22 @@ IEnumerator PublishLocalMicrophoneUnity(Room room) } ``` +#### Unity Audio Processing + +Unity's `Microphone` path does not go through a platform audio device module, so on its own it has no echo cancellation. Pass `AudioProcessingOptions` to run libwebrtc's audio processing (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) over the captured audio before it reaches the track: + +```cs +var processing = new AudioProcessingOptions +{ + EchoCancellation = true, + NoiseSuppression = true, + AutoGainControl = true +}; +var rtcSource = new MicrophoneSource(Microphone.devices[0], microphoneObject, processing); +``` + +Echo cancellation takes its reference from the final mix Unity plays, so it covers every remote `AudioStream` as well as the game's own audio. The SDK attaches a `PlayoutReference` component to the active `AudioListener` for that; adding it to the listener yourself does the same. Unity's output sample rate must be a multiple of 100 Hz (48000, 44100 and 24000 all are); otherwise processing is bypassed with a warning and the raw microphone is published. + #### Unity Audio Output ```cs diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..c18515ff 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -67,4 +67,18 @@ void LiveKit_RestoreDefaultAudioSession() { } } +/// AVAudioSession latency terms, in seconds. Used to seed the echo canceller's stream delay in +/// Unity-audio mode (see AudioProcessingDelayHint.cs). All report 0 until the session is active. +double LiveKit_AudioSessionOutputLatency() { + return [[AVAudioSession sharedInstance] outputLatency]; +} + +double LiveKit_AudioSessionInputLatency() { + return [[AVAudioSession sharedInstance] inputLatency]; +} + +double LiveKit_AudioSessionIOBufferDuration() { + return [[AVAudioSession sharedInstance] IOBufferDuration]; +} + } diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 75bec1f0..48f42868 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -11,6 +11,13 @@ namespace LiveKit /// /// /// Ensure microphone permissions are granted before calling . + /// + /// Unity's Microphone path does not go through a platform audio device module, so on + /// its own it has no echo cancellation. Construct the source with + /// to run libwebrtc's audio processing over the capture; + /// echo cancellation then uses the mix Unity plays as its reference (see + /// ), which covers every remote and the + /// application's own audio. /// sealed public class MicrophoneSource : RtcAudioSource { @@ -35,6 +42,26 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAu _sourceObject = sourceObject; } + /// + /// Creates a microphone source whose capture is run through libwebrtc's audio processing + /// (AEC3 echo cancellation, noise suppression, gain control, high-pass filter) before it + /// reaches the track. + /// + /// The name of the device to capture from. Use to + /// get the list of available devices. + /// The GameObject to attach the AudioSource to. The object must be kept in the scene + /// for the duration of the source's lifetime. + /// Which stages to enable. With + /// the SDK attaches a to the active to obtain the + /// far-end reference. Requires Unity's output sample rate to be a multiple of 100 Hz; otherwise processing is + /// bypassed with a warning. + public MicrophoneSource(string deviceName, GameObject sourceObject, AudioProcessingOptions processing) + : base(RtcAudioSourceType.AudioSourceMicrophone, processing) + { + _deviceName = deviceName; + _sourceObject = sourceObject; + } + /// /// Begins capturing audio from the microphone. /// @@ -207,6 +234,8 @@ private IEnumerator RestartMicrophone() // recover from interruption. Poll for readiness instead of using arbitrary delay. yield return WaitForMicrophoneReady(); + // A resume is a new audio path: drop whatever the processing stage buffered before. + ResetAudioProcessing(); yield return StartMicrophone(); } diff --git a/Runtime/Scripts/Audio/PlatformAudioSource.cs b/Runtime/Scripts/Audio/PlatformAudioSource.cs index 0b507c4c..119adbf1 100644 --- a/Runtime/Scripts/Audio/PlatformAudioSource.cs +++ b/Runtime/Scripts/Audio/PlatformAudioSource.cs @@ -7,7 +7,10 @@ namespace LiveKit { /// - /// Options for audio processing when creating a PlatformAudioSource. + /// Options for libwebrtc's audio processing. Used by , where + /// the ADM applies them, and by Unity-audio sources such as + /// created with options, where the SDK runs libwebrtc's audio processing module over the + /// capture with the mix Unity plays as the echo reference (see ). /// public struct AudioProcessingOptions { @@ -17,7 +20,10 @@ public struct AudioProcessingOptions public bool NoiseSuppression; /// Enable automatic gain control (AGC). Default: true. public bool AutoGainControl; - /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true. + /// + /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + /// only. Default: true. + /// public bool PreferHardware; /// @@ -30,6 +36,9 @@ public struct AudioProcessingOptions AutoGainControl = true, PreferHardware = true }; + + /// Whether any stage of the Unity-audio processing pipeline is enabled. + internal bool AnyProcessingEnabled => EchoCancellation || NoiseSuppression || AutoGainControl; } /// diff --git a/Runtime/Scripts/Audio/Processing.meta b/Runtime/Scripts/Audio/Processing.meta new file mode 100644 index 00000000..180519b6 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: afb866c8c71a34134bf00d5cd4ee1778 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs new file mode 100644 index 00000000..feaf914b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; +using UnityEngine; + +namespace LiveKit +{ + /// + /// Estimates the render-to-capture delay hint handed to + /// . + /// + /// + /// AEC3 runs its own correlation-based delay estimator; the hint only sets the initial + /// alignment after a reset, so it has to be in the right ballpark rather than exact. The echo + /// path in Unity-audio mode is: tap → Unity's output queue (a + /// few DSP blocks) → device output → air → device input → Microphone clip → the + /// AudioSource reading that clip → capture probe. Only the DSP block size and, on iOS, the + /// audio session latencies are readable; the remaining terms are constants. + /// + /// Main thread only: reads . + /// + internal static class AudioProcessingDelayHint + { +#if UNITY_IOS && !UNITY_EDITOR + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionOutputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionInputLatency(); + [DllImport("__Internal")] private static extern double LiveKit_AudioSessionIOBufferDuration(); +#endif + + internal const int MinDelayMs = 0; + internal const int MaxDelayMs = 500; + + /// Output queue depth assumed between the listener tap and the device, in DSP blocks. + internal const int OutputQueueBlocks = 2; + + /// + /// How far the AudioSource reading the microphone clip trails the clip's write head. + /// starts reading once Microphone.GetPosition first + /// reports data, polled at 50 ms, and that offset persists for the life of the clip. + /// + internal const int MicrophoneReadBehindMs = 50; + + /// Device input plus output latency when the platform does not report it. + internal const int FallbackDeviceLatencyMs = 30; + + public static int EstimateMs() + { + var config = AudioSettings.GetConfiguration(); + return EstimateMs(config.dspBufferSize, config.sampleRate, PlatformLatencyMs()); + } + + internal static int EstimateMs(int dspBufferSize, int sampleRate, double deviceLatencyMs) + { + var blockMs = sampleRate > 0 ? dspBufferSize * 1000.0 / sampleRate : 0.0; + var estimate = blockMs * OutputQueueBlocks + deviceLatencyMs + MicrophoneReadBehindMs; + return (int)Math.Round(Math.Min(MaxDelayMs, Math.Max(MinDelayMs, estimate))); + } + + internal static double PlatformLatencyMs() + { +#if UNITY_IOS && !UNITY_EDITOR + try + { + // AVAudioSession reports zero until the session is active, hence the fallback. + var output = LiveKit_AudioSessionOutputLatency(); + var input = LiveKit_AudioSessionInputLatency(); + var ioBuffer = LiveKit_AudioSessionIOBufferDuration(); + if (output > 0d || input > 0d) + return (output + input + ioBuffer) * 1000d; + } + catch (Exception) + { + // Fall through to the constant. + } +#endif + return FallbackDeviceLatencyMs; + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta new file mode 100644 index 00000000..c97cd183 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingDelayHint.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b1bb08510396f4e28b12569471ea0cf6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs new file mode 100644 index 00000000..5e264a2b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs @@ -0,0 +1,167 @@ +using System; +using LiveKit.Internal.FFI; +using LiveKit.Internal.FFI.Requests; +using LiveKit.Proto; + +namespace LiveKit +{ + /// + /// libwebrtc's AudioProcessingModule (AEC3 echo cancellation, noise suppression, gain + /// control, high-pass filter), driven over the FFI. + /// + /// + /// Use this to run echo cancellation over a capture path that does not go through the + /// platform audio device module (e.g. Unity's Microphone): feed the audio that is + /// played out of the loudspeaker to and the captured + /// microphone audio to , which processes it in place. + /// + /// Both accept exactly one 10 ms chunk of interleaved int16 PCM ( + /// samples per channel) and nothing else. libwebrtc's own contract is a capture thread calling + /// and a render thread calling ; + /// the native module is internally synchronised for exactly that split, and the SDK's request + /// plumbing is safe to use from both. + /// + internal sealed class AudioProcessingModule : IDisposable + { + /// libwebrtc's kChunkSizeMs — the APM accepts nothing else. + public const int ChunkSizeMs = 10; + + /// + /// libwebrtc's internal processing rates. The APM resamples any other API rate onto one of + /// these itself, so this list is diagnostic information — NOT an admission requirement. See + /// . + /// + private static readonly int[] NativeSampleRates = { 8000, 16000, 32000, 48000 }; + + private readonly FfiHandle _handle; + private bool _disposed; + + /// The native handle id, for diagnostics. + public ulong Handle => (ulong)_handle.DangerousGetHandle(); + + public AudioProcessingModule( + bool echoCancellerEnabled, + bool gainControllerEnabled, + bool highPassFilterEnabled, + bool noiseSuppressionEnabled) + { + using var request = FFIBridge.Instance.NewRequest(); + var newApm = request.request; + newApm.EchoCancellerEnabled = echoCancellerEnabled; + newApm.GainControllerEnabled = gainControllerEnabled; + newApm.HighPassFilterEnabled = highPassFilterEnabled; + newApm.NoiseSuppressionEnabled = noiseSuppressionEnabled; + + using var response = request.Send(); + FfiResponse res = response; + var owned = res.NewApm?.Apm; + if (owned?.Handle == null || owned.Handle.Id == 0) + throw new InvalidOperationException("FFI returned no APM handle"); + + _handle = FfiHandle.FromOwnedHandle(owned.Handle); + } + + public static bool IsNativeSampleRate(int sampleRate) + { + foreach (var rate in NativeSampleRates) + if (rate == sampleRate) return true; + return false; + } + + /// + /// Whether the APM accepts this rate on its API surface. + /// + /// + /// The only hard requirement is that one 10 ms chunk is a whole number of samples: both + /// here and libwebrtc's own StreamConfig::num_frames() + /// derive the chunk with integer division, so a rate that is not a multiple of 100 Hz would + /// short every chunk and drift the two feeds apart. + /// + /// A non-native rate is NOT rejected — the rate goes straight into a StreamConfig + /// and libwebrtc resamples to a native processing rate internally. A 24 kHz output rate + /// (iPad) is cancelled just as well as 48 kHz. + /// + public static bool IsSupportedApiRate(int sampleRate) => + sampleRate > 0 && sampleRate % (1000 / ChunkSizeMs) == 0; + + /// Samples per channel in one APM chunk at the given rate. + public static int FrameSizeFor(int sampleRate) => sampleRate / (1000 / ChunkSizeMs); + + /// + /// Processes the near-end (capture) stream in place. is bytes, + /// not samples — the buffer is interleaved int16. Returns the FFI error, or null on success. + /// + public string ProcessStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var process = request.request; + process.ApmHandle = Handle; + process.DataPtr = (ulong)dataPtr.ToInt64(); + process.Size = (uint)byteCount; + process.SampleRate = (uint)sampleRate; + process.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessStream?.Error); + } + + /// + /// Processes the far-end (render) reference stream in place. Same buffer contract as + /// . + /// + public string ProcessReverseStream(IntPtr dataPtr, int byteCount, int sampleRate, int channels) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var reverse = request.request; + reverse.ApmHandle = Handle; + reverse.DataPtr = (ulong)dataPtr.ToInt64(); + reverse.Size = (uint)byteCount; + reverse.SampleRate = (uint)sampleRate; + reverse.NumChannels = (uint)channels; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmProcessReverseStream?.Error); + } + + /// + /// Seeds the render/capture delay. AEC3 runs its own correlation estimator, so this is a + /// convergence hint rather than a hard alignment. Returns the FFI error, or null on success. + /// + public string SetStreamDelayMs(int delayMs) + { + if (_disposed) throw new ObjectDisposedException(nameof(AudioProcessingModule)); + + using var request = FFIBridge.Instance.NewRequest(); + var delay = request.request; + delay.ApmHandle = Handle; + delay.DelayMs = delayMs; + + using var response = request.Send(); + FfiResponse res = response; + return ErrorOrNull(res.ApmSetStreamDelay?.Error); + } + + private static string ErrorOrNull(string error) => string.IsNullOrEmpty(error) ? null : error; + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + GC.SuppressFinalize(this); + } + + ~AudioProcessingModule() + { + if (_disposed) return; + _disposed = true; + _handle.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta new file mode 100644 index 00000000..4263c783 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessingModule.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 033fc8514082a43a18e018ca3267a9b5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs new file mode 100644 index 00000000..f77a6f38 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs @@ -0,0 +1,343 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Threading; +using LiveKit.Internal; +using LiveKit.Internal.FFI; +using LiveKit.Internal.Threading; +using UnityEngine; + +namespace LiveKit +{ + /// + /// The processing stage behind on an + /// : owns one , re-chunks the + /// capture and the playout reference into 10 ms frames, and hands every processed capture + /// chunk to the source for the FFI. + /// + /// + /// Threading. (from the source's audio callback) and + /// (from ) both run on the Unity + /// audio thread. Within one DSP tick the capture probes run before the listener tap, so the + /// reference for tick N arrives after the capture of tick N; that is fine because the acoustic + /// echo of tick N's playout only reaches the microphone several ticks later. , + /// and the maintenance coroutine run on the main thread and never touch the + /// ring buffers; they raise flags the audio thread acts on. Nothing here logs from the audio + /// thread; the two warnings it can raise are posted to the main thread. + /// + /// The Rust side asserts (and takes the process down) on a frame that is not a whole multiple + /// of 10 ms, so the chunking here is not optional, and blocks at a rate whose 10 ms chunk is + /// not a whole number of samples are bypassed. + /// + internal sealed class AudioProcessor : IDisposable + { + /// Receives one processed 10 ms chunk. The frame is valid only during the call. + internal delegate void ProcessedFrameSink(ReadOnlySpan frame, int channels, int sampleRate); + + // Ring capacity in chunks. Bounds the latency added when a DSP block is not a multiple of 10 ms. + private const int BufferedChunks = 8; + private const float MaintenanceIntervalSeconds = 2f; + + private readonly AudioProcessingModule _apm; + private readonly ProcessedFrameSink _sink; + private readonly bool _echoCancellation; + + // Guards the pinned reference chunk against Dispose racing an in-flight callback. The + // capture side needs no lock: its chunk is pinned only for the duration of each call. + private readonly object _referenceLock = new object(); + + // Capture side. Audio thread only. + private PcmRingBuffer _captureRing; + private short[] _captureStaging; + private short[] _captureChunk; + private int _captureRate; + private int _captureChannels; + private int _captureChunkSamples; + + // Reference side. Audio thread only, under _referenceLock. The chunk stays pinned so the + // module has a stable address to process in place. + private PcmRingBuffer _referenceRing; + private short[] _referenceStaging; + private short[] _referenceChunk; + private GCHandle _referenceChunkPin; + private int _referenceRate; + private int _referenceChannels; + private int _referenceChunkSamples; + + // Cross-thread flags. + private volatile bool _running; + private volatile bool _disposed; + private int _captureResetRequested; + private int _referenceResetRequested; + private int _unsupportedRateWarned; + private int _moduleFailureWarned; + private int _maintenanceGeneration; + + // Main thread only. + private int _delayHintMs = -1; + + /// The FFI could not create the module. + public AudioProcessor(AudioProcessingOptions options, ProcessedFrameSink sink) + { + _sink = sink ?? throw new ArgumentNullException(nameof(sink)); + _echoCancellation = options.EchoCancellation; + // The high-pass filter is not exposed as an option: libwebrtc instantiates it anyway + // whenever AEC or NS is on, and the platform ADM path always runs it, so both paths + // stay identical. + _apm = new AudioProcessingModule( + echoCancellerEnabled: options.EchoCancellation, + gainControllerEnabled: options.AutoGainControl, + highPassFilterEnabled: true, + noiseSuppressionEnabled: options.NoiseSuppression); + } + + /// Main thread. + public void Start() + { + if (_disposed || _running) return; + _running = true; + RequestReset(); + + if (_echoCancellation) + PlayoutReference.Acquire(OnPlayoutAudio); + + SeedDelayHint(); + // Only echo cancellation has periodic upkeep. The loop never finishes on its own, so it + // is not handed to a host that would drain it synchronously. + if (_echoCancellation && MonoBehaviourContext.CanRunCoroutines) + MonoBehaviourContext.RunCoroutine(Maintenance(++_maintenanceGeneration)); + } + + /// Main thread. + public void Stop() + { + if (!_running) return; + _running = false; + + if (_echoCancellation) + PlayoutReference.Release(OnPlayoutAudio); + } + + /// + /// Any thread. Clears both feeds before their next audio callback. Call when the capture + /// path restarts (e.g. a microphone resume) so stale samples do not misalign the canceller. + /// + public void RequestReset() + { + Interlocked.Exchange(ref _captureResetRequested, 1); + Interlocked.Exchange(ref _referenceResetRequested, 1); + } + + // Periodic main-thread upkeep for echo cancellation: re-attach the reference after scene or + // device changes and refresh the delay hint (iOS reports zero session latency until the + // session is active). Unscaled time, so a paused game keeps its reference. + private IEnumerator Maintenance(int generation) + { + while (_running && !_disposed && generation == _maintenanceGeneration) + { + PlayoutReference.EnsureAttached(); + SeedDelayHint(); + yield return new WaitForSecondsRealtime(MaintenanceIntervalSeconds); + } + } + + private void SeedDelayHint() + { + if (!_echoCancellation || _disposed) return; + + var hint = AudioProcessingDelayHint.EstimateMs(); + if (hint == _delayHintMs) return; + + string error; + try + { + error = _apm.SetStreamDelayMs(hint); + } + catch (ObjectDisposedException) + { + return; + } + + if (error != null) + { + Utils.Warning($"AudioProcessor: set_stream_delay_ms({hint}) failed: {error}"); + return; + } + + _delayHintMs = hint; + } + + /// + /// Unity audio thread. Runs the block through the module in 10 ms chunks and forwards each + /// chunk to the sink. Returns false when the block must go out unprocessed instead: + /// processing is stopped, or bypassed for this sample rate. + /// + public bool TryProcessCapture(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running) return false; + if (data == null || data.Length == 0 || channels <= 0 || sampleRate <= 0) return false; + + // Checked per block rather than latched: Unity's output rate can change with the + // device, and a rate that becomes supported later is processed again. + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) + { + WarnUnsupportedRate(sampleRate); + return false; + } + + if (Interlocked.Exchange(ref _captureResetRequested, 0) == 1) + _captureRing?.Clear(); + + if (_captureStaging == null || sampleRate != _captureRate || channels != _captureChannels || _captureStaging.Length < data.Length) + ConfigureCapture(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _captureStaging[i] = PcmConvert.FloatToS16(data[i]); + _captureRing.Write(_captureStaging, 0, data.Length); + + while (_captureRing.TryDrain(_captureChunk, _captureChunkSamples)) + { + ProcessCaptureChunk(_captureChunk, sampleRate, channels); + _sink(_captureChunk, channels, sampleRate); + } + + return true; + } + + private void ProcessCaptureChunk(short[] chunk, int sampleRate, int channels) + { + try + { + string error; + unsafe + { + fixed (short* ptr = chunk) + error = _apm.ProcessStream((IntPtr)ptr, chunk.Length * sizeof(short), sampleRate, channels); + } + if (error != null) WarnModuleFailure(error); + } + catch (Exception e) + { + // The chunk goes out unprocessed rather than not at all. + WarnModuleFailure(e.Message); + } + } + + // Unity audio thread, from PlayoutReference. Must not modify data. + private void OnPlayoutAudio(float[] data, int channels, int sampleRate) + { + if (_disposed || !_running) return; + if (data == null || data.Length == 0 || channels <= 0) return; + if (!AudioProcessingModule.IsSupportedApiRate(sampleRate)) return; + + lock (_referenceLock) + { + if (_disposed) return; + + if (Interlocked.Exchange(ref _referenceResetRequested, 0) == 1) + _referenceRing?.Clear(); + + if (_referenceStaging == null || sampleRate != _referenceRate || channels != _referenceChannels || _referenceStaging.Length < data.Length) + ConfigureReference(sampleRate, channels, data.Length); + + for (var i = 0; i < data.Length; i++) + _referenceStaging[i] = PcmConvert.FloatToS16(data[i]); + _referenceRing.Write(_referenceStaging, 0, data.Length); + + var byteCount = _referenceChunkSamples * sizeof(short); + while (_referenceRing.TryDrain(_referenceChunk, _referenceChunkSamples)) + { + try + { + var error = _apm.ProcessReverseStream(_referenceChunkPin.AddrOfPinnedObject(), byteCount, sampleRate, channels); + if (error != null) WarnModuleFailure(error); + } + catch (Exception e) + { + WarnModuleFailure(e.Message); + } + } + } + } + + // Format changes are rare (first block, device switch); the allocations here are accepted + // on the audio thread for the same reason AudioStream sizes its buffers lazily. + private void ConfigureCapture(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _captureRate = sampleRate; + _captureChannels = channels; + _captureChunkSamples = chunkSamples; + _captureChunk = new short[chunkSamples]; + _captureStaging = new short[incomingSamples]; + // Never smaller than one input block, or a large block would overflow immediately. + _captureRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + private void ConfigureReference(int sampleRate, int channels, int incomingSamples) + { + var chunkSamples = AudioProcessingModule.FrameSizeFor(sampleRate) * channels; + _referenceRate = sampleRate; + _referenceChannels = channels; + _referenceChunkSamples = chunkSamples; + + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = new short[chunkSamples]; + _referenceChunkPin = GCHandle.Alloc(_referenceChunk, GCHandleType.Pinned); + + _referenceStaging = new short[incomingSamples]; + _referenceRing = new PcmRingBuffer(Math.Max(chunkSamples * BufferedChunks, incomingSamples + chunkSamples)); + } + + // The module rejected a chunk, which went out unprocessed. Warn once. + private void WarnModuleFailure(string error) + { + if (Interlocked.Exchange(ref _moduleFailureWarned, 1) == 1) return; + + PostWarning("AudioProcessor: the audio processing module rejected a chunk, which was published " + + $"unprocessed: {error}"); + } + + private void WarnUnsupportedRate(int sampleRate) + { + if (Interlocked.Exchange(ref _unsupportedRateWarned, 1) == 1) return; + + PostWarning($"AudioProcessor: Unity's output sample rate {sampleRate} Hz has no whole-sample 10 ms chunk; " + + "audio processing is bypassed and the capture is published unprocessed."); + } + + // Logging is not allowed on the audio thread; hand the message to the main thread. + private static void PostWarning(string message) + { + var context = FfiClient.Instance._context; + if (context != null) + context.Post(static m => Utils.Warning(m), message); + else + Utils.Warning(message); + } + + public void Dispose() => Dispose(true); + + /// + /// is false on the owner's finalizer path: only the native + /// module and the pinned chunk are released there. The + /// registration is main-thread state and is only touched by . + /// + internal void Dispose(bool disposing) + { + if (_disposed) return; + + if (disposing) Stop(); + else _running = false; + + lock (_referenceLock) + { + _disposed = true; + if (_referenceChunkPin.IsAllocated) _referenceChunkPin.Free(); + _referenceChunk = null; + _referenceRing = null; + } + _apm.Dispose(); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta new file mode 100644 index 00000000..593e25f5 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/AudioProcessor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0154e5d4b40264653a6b7832d5640b7a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs new file mode 100644 index 00000000..a9acddbb --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs @@ -0,0 +1,118 @@ +using System; + +namespace LiveKit +{ + /// + /// Fixed-capacity interleaved int16 PCM ring buffer with a fixed-size drain. Re-chunks Unity's + /// DSP-block-sized audio into the 10 ms frames the requires. + /// + /// + /// Allocation-free after construction: both users run on the Unity audio thread. Sized in + /// samples (frames × channels), not frames. Single producer and single consumer per instance; + /// the capture and reference feeds each own one, so there is no synchronisation inside. + /// + internal sealed class PcmRingBuffer + { + private readonly short[] _buffer; + private int _readIndex; + private int _writeIndex; + private int _count; + + /// Samples dropped because the buffer was full, since construction. + public int OverflowSamples { get; private set; } + + public int Capacity => _buffer.Length; + public int Available => _count; + + public PcmRingBuffer(int capacitySamples) + { + if (capacitySamples <= 0) throw new ArgumentOutOfRangeException(nameof(capacitySamples)); + _buffer = new short[capacitySamples]; + } + + /// + /// Appends samples. When the buffer is full the OLDEST samples are + /// dropped: a stalled consumer must not push the echo reference arbitrarily far out of + /// alignment with the capture stream. + /// + public void Write(short[] source, int offset, int count) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (offset < 0 || count < 0 || offset + count > source.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (count >= _buffer.Length) + { + OverflowSamples += _count + count - _buffer.Length; + offset += count - _buffer.Length; + count = _buffer.Length; + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + else + { + var free = _buffer.Length - _count; + if (count > free) Discard(count - free); + } + + for (var i = 0; i < count; i++) + { + _buffer[_writeIndex] = source[offset + i]; + _writeIndex = _writeIndex + 1 == _buffer.Length ? 0 : _writeIndex + 1; + } + + _count += count; + } + + /// + /// Copies exactly samples into and + /// consumes them. Returns false and consumes nothing when fewer are available. + /// + public bool TryDrain(short[] destination, int count) + { + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (count < 0 || count > destination.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (_count < count) return false; + + for (var i = 0; i < count; i++) + { + destination[i] = _buffer[_readIndex]; + _readIndex = _readIndex + 1 == _buffer.Length ? 0 : _readIndex + 1; + } + + _count -= count; + return true; + } + + public void Clear() + { + _readIndex = 0; + _writeIndex = 0; + _count = 0; + } + + private void Discard(int count) + { + if (count > _count) count = _count; + _readIndex = (_readIndex + count) % _buffer.Length; + _count -= count; + OverflowSamples += count; + } + } + + /// Sample format conversions shared by the capture paths. + internal static class PcmConvert + { + /// Float [-1, 1] to int16 with clamping and round-half-away-from-zero. + public static short FloatToS16(float v) + { + v *= 32768f; + if (v > 32767f) v = 32767f; + else if (v < -32768f) v = -32768f; + return (short)(v + Math.Sign(v) * 0.5f); + } + + public static float S16ToFloat(short v) => v / 32768f; + } +} diff --git a/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta new file mode 100644 index 00000000..f096bd2b --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PcmRingBuffer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cce610429285242e193622fd7e37bb80 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs new file mode 100644 index 00000000..fd45d081 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs @@ -0,0 +1,165 @@ +using System.Collections; +using UnityEngine; +using LiveKit.Internal; +using LiveKit.Internal.Threading; + +namespace LiveKit +{ + /// + /// Taps the final mix Unity sends to the audio output device and feeds it to the echo + /// canceller as the far-end reference. + /// + /// Attaches itself to the GameObject of the active , the virtual + /// microphone in the scene, usually on the camera, which hears the scene and sends the result + /// to the audio output hardware. + /// + /// + /// An created with + /// attaches this component to the active listener when it starts and re-attaches it after + /// scene loads and audio device changes. Adding it to the listener yourself is supported and + /// does the same thing. + /// + /// Because the tap sits after every AudioSource, mixer group and spatializer, the reference is + /// exactly what the loudspeaker plays: every remote participant plus the game's own audio. + /// The capture probe clears its buffer after reading, so the + /// local microphone never appears in the mix. + /// + /// OnAudioFilterRead runs on the Unity audio thread and must not touch Unity APIs, so + /// the sample rate, listener state and consumer count are cached on the main thread. + /// + [DisallowMultipleComponent] + public sealed class PlayoutReference : MonoBehaviour + { + internal delegate void PlayoutAudioDelegate(float[] data, int channels, int sampleRate); + + // Raised on the Unity audio thread with the final mix. Consumers must not modify the + // buffer: it is on its way to the speaker. The invocation list is the consumer list, so + // a consumer cannot exist without holding the reference and vice versa. + private static event PlayoutAudioDelegate AudioRead; + private static bool HasConsumers => AudioRead != null; + + // Singleton pattern instance + private static PlayoutReference _instance; + + // The AudioListener we are attached to + private AudioListener _sceneAudioListener; + private volatile int _sampleRate; + private volatile bool _deliver; + + /// Whether a reference on an enabled listener is delivering audio to a consumer. + internal static bool IsAttached => _instance != null && _instance._deliver; + + /// + /// Main thread. Registers to receive the final mix and attaches + /// to the listener if possible. Acquiring the same consumer twice delivers to it twice + /// until it is released twice. + /// + internal static void Acquire(PlayoutAudioDelegate consumer) + { + AudioRead += consumer; + EnsureAttached(); + if (_instance != null) _instance.RefreshDeliveryState(); + } + + /// + /// Main thread. Removes . The component stays on the listener; + /// once the last consumer is gone it stops delivering. Releasing a consumer that was not + /// acquired is a no-op. + /// + internal static void Release(PlayoutAudioDelegate consumer) + { + AudioRead -= consumer; + if (_instance != null) _instance.RefreshDeliveryState(); + } + + /// + /// Main thread. Attaches to the active AudioListener unless a working reference already + /// exists. No-op without consumers or without a listener; consumers call this periodically, + /// which is what covers scene loads and a destroyed listener. + /// + internal static void EnsureAttached() + { + if (!HasConsumers) return; + if (_instance != null && _instance.isActiveAndEnabled && + _instance._sceneAudioListener != null && _instance._sceneAudioListener.isActiveAndEnabled) + return; + + var listener = FindActiveListener(); + if (listener == null) return; + + var existing = listener.GetComponent(); + _instance = existing != null ? existing : listener.gameObject.AddComponent(); + } + + private static AudioListener FindActiveListener() + { + var listeners = FindObjectsByType(FindObjectsSortMode.None); + foreach (var listener in listeners) + { + if (listener.isActiveAndEnabled) return listener; + } + return null; + } + + private void OnEnable() + { + _sceneAudioListener = GetComponent(); + if (_sceneAudioListener == null) + Utils.Warning("PlayoutReference must be on the AudioListener's GameObject; it will not deliver a reference from here."); + + RefreshDeliveryState(); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + if (_instance == null) _instance = this; + } + + private void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + _deliver = false; + if (_instance == this) _instance = null; + } + + private void Update() + { + RefreshDeliveryState(); + } + + // Listener state and the output rate are Unity APIs and the consumer list is mutated on the + // main thread; fold them into one flag so the audio thread reads a single volatile. + private void RefreshDeliveryState() + { + _sampleRate = AudioSettings.outputSampleRate; + _deliver = HasConsumers && _sceneAudioListener != null && _sceneAudioListener.isActiveAndEnabled; + } + + // Unity rebuilds the DSP graph on a device change (or AudioSettings.Reset), which can leave + // filter nodes detached; AudioStream recreates its probe for the same reason. Recreate this + // component so the tap is registered on the new graph. Only done while something consumes + // the reference, so a hand-placed component in an idle scene is left alone. + private void OnAudioConfigurationChanged(bool deviceWasChanged) + { + RefreshDeliveryState(); + if (!HasConsumers) return; + + var host = gameObject; + Destroy(this); + MonoBehaviourContext.RunCoroutine(Reattach(host)); + } + + private static IEnumerator Reattach(GameObject host) + { + // Let the deferred Destroy apply before adding the replacement. + yield return null; + if (host == null || !HasConsumers) yield break; + if (host.GetComponent() == null) + _instance = host.AddComponent(); + } + + // Unity audio thread. + private void OnAudioFilterRead(float[] data, int channels) + { + if (!_deliver) return; + AudioRead?.Invoke(data, channels, _sampleRate); + } + } +} diff --git a/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta new file mode 100644 index 00000000..a74f1070 --- /dev/null +++ b/Runtime/Scripts/Audio/Processing/PlayoutReference.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 44cbde2518fe54f5ebd5f24f65d8a070 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Audio/RtcAudioSource.cs b/Runtime/Scripts/Audio/RtcAudioSource.cs index 9147b431..23891eab 100644 --- a/Runtime/Scripts/Audio/RtcAudioSource.cs +++ b/Runtime/Scripts/Audio/RtcAudioSource.cs @@ -1,12 +1,8 @@ using System; using System.Collections; -using System.Collections.Generic; using LiveKit.Proto; using LiveKit.Internal; using LiveKit.Internal.FFI.Requests; -using Unity.Collections; -using Unity.Collections.LowLevel.Unsafe; -using System.Diagnostics; using System.Threading; using LiveKit.Internal.FFI; @@ -26,16 +22,6 @@ public enum RtcAudioSourceType /// public abstract class RtcAudioSource : IRtcSource, IDisposable { - private sealed class PendingAudioFrame - { - public NativeArray FrameData; - public int FrameIndex; - public int SampleRate; - public int Channels; - public int SampleCount; - public long StartedTimestamp; - } - private static int nextDebugId = 0; /// @@ -43,28 +29,37 @@ private sealed class PendingAudioFrame /// Provides the audio data, channel count, and sample rate. /// /// - /// This event is not guaranteed to be called on the main thread. + /// This event is not guaranteed to be called on the main thread. It must not be invoked + /// concurrently: the source converts each block into one reusable buffer. /// public abstract event Action AudioRead; private readonly RtcAudioSourceType _sourceType; public RtcAudioSourceType SourceType => _sourceType; + + /// + /// Whether this source runs libwebrtc's audio processing over its capture. False when it was + /// created without , or when the module could not be + /// created and the source fell back to unprocessed capture. + /// + public bool AudioProcessingEnabled => _processor != null; private readonly int _debugId = Interlocked.Increment(ref nextDebugId); internal readonly uint _expectedSampleRate; internal readonly uint _expectedChannels; internal readonly FfiHandle Handle; protected AudioSourceInfo _info; + private readonly AudioProcessor _processor; - // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM - // pointer after request.Send() returns and encode it later on another queue. Because of - // that, a single reusable NativeArray is unsafe here; the next AudioRead callback can - // overwrite it while Opus/WebRTC is still consuming the previous frame. - // - // Keep one NativeArray per in-flight request and release it only after the matching - // CaptureAudioFrame callback completes or is canceled. - private readonly Dictionary _pendingFrameData = new(); - private readonly object _pendingFrameDataLock = new object(); + // CaptureAudioFrame copies the PCM into its own buffer on the calling thread before the + // request returns (livekit-ffi capture_frame, rust-sdks #289), so the pointer only has to + // stay valid for the duration of the synchronous Send(). One reusable buffer, pinned around + // the call, is enough. Audio thread only; see the AudioRead contract. + private short[] _captureBuffer = Array.Empty(); + + // Cached so a capture registers its callback without allocating per frame. + private readonly Action _onCaptureCallback; + private readonly Action _onCaptureCanceled; private volatile bool _muted = false; public override bool Muted => _muted; @@ -72,18 +67,33 @@ private sealed class PendingAudioFrame private bool _started = false; private volatile bool _disposed = false; private int _audioReadCount = 0; + private int _sentFrameCount = 0; // Device-capture sources (microphone, AudioSource taps) don't know their format ahead of // time — it is whatever Unity's audio graph delivers. They use this constructor, which // configures the native source from Unity's current output configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType) - : this(audioSourceType, 0, 0) { } + : this(audioSourceType, 0, 0, null) { } + + /// + /// Device-capture source whose audio is run through libwebrtc's audio processing (echo + /// cancellation, noise suppression, gain control, high-pass filter) before it reaches the + /// track. See . If the module cannot be created the + /// source logs a warning and captures unprocessed. + /// + protected RtcAudioSource(RtcAudioSourceType audioSourceType, AudioProcessingOptions processing) + : this(audioSourceType, 0, 0, processing) { } // Sources that generate a fixed, known format (e.g. test signal generators) declare it // directly. Passing 0 for either value falls back to the device configuration. protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels) + : this(audioSourceType, sampleRate, channels, null) { } + + protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, uint channels, AudioProcessingOptions? processing) { _sourceType = audioSourceType; + _onCaptureCallback = OnCaptureCallback; + _onCaptureCanceled = OnCaptureCanceled; if (sampleRate > 0 && channels > 0) { @@ -110,22 +120,49 @@ protected RtcAudioSource(RtcAudioSourceType audioSourceType, uint sampleRate, ui _info = res.NewAudioSource.Source.Info; Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); Utils.Debug($"{DebugTag} created handle={Handle.DangerousGetHandle()} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + + if (processing is { } options && options.AnyProcessingEnabled) + { + try + { + _processor = new AudioProcessor(options, SendProcessedFrame); + } + catch (Exception e) + { + // Publish unprocessed rather than not at all. + Utils.Warning($"{DebugTag} audio processing unavailable, capturing unprocessed: {e.Message}"); + } + } } + // Format used when Unity reports no usable output configuration. Matches the FFI defaults. + private const uint FallbackSampleRate = 48000; + private const uint FallbackChannels = 1; + // Reads Unity's actual output audio configuration. The capture path delivers buffers at the // DSP output rate/channel count (see AudioProbe), so this is the format the native source - // must match. Falls back to the platform defaults when Unity cannot report a configuration - // (e.g. batch mode without an audio device). + // must match. When the Unity audio system is disabled (Project Settings > Audio > Disable + // Unity Audio, dedicated servers) Unity reports a 0 Hz rate and the Raw speaker mode. The + // native source must not be created with that format: its 10 ms silence timer divides by + // the channel count once the track is published. Fall back to the FFI defaults and warn; + // capture through Unity audio cannot work in that state, but the process stays alive. private (uint sampleRate, uint channels) ResolveDeviceFormat() { var config = UnityEngine.AudioSettings.GetConfiguration(); - var sampleRate = (uint)config.sampleRate; - var configuredChannels = SpeakerModeChannels(config.speakerMode); - var channels = configuredChannels; + var sampleRate = config.sampleRate; + var channels = SpeakerModeChannels(config.speakerMode); + + if (sampleRate <= 0 || channels == 0) + { + Utils.Warning($"{DebugTag} Unity reports no usable output format (sampleRate={sampleRate}, " + + $"speakerMode={config.speakerMode}); the Unity audio system is probably disabled. " + + $"Falling back to {FallbackSampleRate} Hz, {FallbackChannels} channel(s)."); + return (FallbackSampleRate, FallbackChannels); + } Utils.Info($"Configured native audio source with sampleRate {sampleRate} and channels {channels}"); - return (sampleRate, channels); + return ((uint)sampleRate, channels); } private static uint SpeakerModeChannels(UnityEngine.AudioSpeakerMode mode) @@ -150,6 +187,7 @@ public virtual void Start() { if (_started) return; AudioRead += OnAudioRead; + _processor?.Start(); _started = true; Utils.Debug($"{DebugTag} start"); } @@ -161,137 +199,113 @@ public virtual void Stop() { if (!_started) return; AudioRead -= OnAudioRead; + _processor?.Stop(); _started = false; - var pendingCount = PendingFrameCount(); - if (pendingCount > 0) - Utils.Warning($"{DebugTag} stop requested with {pendingCount} pending capture callbacks"); - else - Utils.Debug($"{DebugTag} stop"); + Utils.Debug($"{DebugTag} stop"); } private void OnAudioRead(float[] data, int channels, int sampleRate) { - if (_muted) return; if (_disposed) return; + // A muted block still runs through the processing stage so the echo canceller keeps + // seeing the near end next to its reference; SendProcessedFrame drops the output. + // Without a processing stage there is nothing to keep warm. + if (_muted && _processor == null) return; - var frameIndex = Interlocked.Increment(ref _audioReadCount); + var readIndex = Interlocked.Increment(ref _audioReadCount); if (channels <= 0) { - Utils.Warning($"{DebugTag} dropping audio frame #{frameIndex} because channels={channels}"); + Utils.Warning($"{DebugTag} dropping audio frame #{readIndex} because channels={channels}"); return; } if (data.Length == 0 || data.Length % channels != 0) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} has invalid shape samples={data.Length} channels={channels}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} has invalid shape samples={data.Length} channels={channels}"); return; } if ((uint)sampleRate != _expectedSampleRate || (uint)channels != _expectedChannels) { - Utils.Warning($"{DebugTag} audio frame #{frameIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + Utils.Warning($"{DebugTag} audio frame #{readIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); } - var pendingBeforeSend = PendingFrameCount(); - if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) - { - Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={data.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); - } + // Optional processing stage: the block is re-chunked into 10 ms frames, run through the + // module and delivered to SendFrame one chunk at a time via SendProcessedFrame. + if (_processor != null && _processor.TryProcessCapture(data, channels, sampleRate)) + return; - // Each captured frame gets its own backing buffer so the native encoder can safely - // consume it asynchronously after request.Send() returns. - var frameData = new NativeArray(data.Length, Allocator.Persistent); + if (_muted) return; + + if (_captureBuffer.Length < data.Length) + _captureBuffer = new short[data.Length]; + for (int i = 0; i < data.Length; i++) + _captureBuffer[i] = PcmConvert.FloatToS16(data[i]); + + SendFrame(new ReadOnlySpan(_captureBuffer, 0, data.Length), channels, sampleRate); + } - // Copy from the audio read buffer into the frame buffer, converting - // each sample to a 16-bit signed integer. - static short FloatToS16(float v) + // Audio thread, from the processing stage. Muted output is dropped here, after the module + // has seen the block. + private void SendProcessedFrame(ReadOnlySpan frame, int channels, int sampleRate) + { + if (_disposed || _muted) return; + SendFrame(frame, channels, sampleRate); + } + + // Hands one int16 frame to the native source. The frame is borrowed for the duration of + // the call only; the native side has its own copy when Send() returns. + private unsafe void SendFrame(ReadOnlySpan frame, int channels, int sampleRate) + { + var frameIndex = Interlocked.Increment(ref _sentFrameCount); + if (frameIndex <= 3 || frameIndex % 100 == 0) { - v *= 32768f; - v = Math.Min(v, 32767f); - v = Math.Max(v, -32768f); - return (short)(v + Math.Sign(v) * 0.5f); + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={frame.Length} channels={channels} sampleRate={sampleRate} thread={Thread.CurrentThread.ManagedThreadId}"); } - for (int i = 0; i < data.Length; i++) - frameData[i] = FloatToS16(data[i]); - // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); using var audioFrameBufferInfo = request.TempResource(); var pushFrame = request.request; pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); pushFrame.Buffer = audioFrameBufferInfo; - unsafe - { - pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility - .GetUnsafePtr(frameData); - } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; - pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; - - // Wait for async callback, log an error if the capture fails. The callback's AsyncId - // echoes the RequestAsyncId that Unity wrote onto the request. - var requestAsyncId = request.RequestAsyncId; - var pendingFrame = new PendingAudioFrame - { - FrameData = frameData, - FrameIndex = frameIndex, - SampleRate = sampleRate, - Channels = channels, - SampleCount = data.Length, - StartedTimestamp = Stopwatch.GetTimestamp(), - }; - lock (_pendingFrameDataLock) - { - _pendingFrameData[requestAsyncId] = pendingFrame; - } + pushFrame.Buffer.SamplesPerChannel = (uint)(frame.Length / channels); - void Callback(CaptureAudioFrameCallback callback) - { - if (callback.AsyncId != requestAsyncId) return; - var completedFrame = ReleasePendingFrameData(requestAsyncId); - if (completedFrame != null) - { - var elapsedMs = ElapsedMilliseconds(completedFrame.StartedTimestamp); - if (callback.HasError) - { - Utils.Error($"{DebugTag} capture callback failed asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()} error={callback.Error}"); - } - else if (completedFrame.FrameIndex <= 3 || completedFrame.FrameIndex % 100 == 0 || elapsedMs > 100) - { - Utils.Debug($"{DebugTag} capture callback asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); - } - } - if (callback.HasError) - Utils.Error($"{DebugTag} audio capture failed: {callback.Error}"); - } - void OnCanceled() - { - var canceledFrame = ReleasePendingFrameData(requestAsyncId); - if (canceledFrame != null) - { - var elapsedMs = ElapsedMilliseconds(canceledFrame.StartedTimestamp); - Utils.Warning($"{DebugTag} capture callback canceled asyncId={requestAsyncId} frame={canceledFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); - } - } + // The callback only reports the outcome. It is registered before Send() so Rust cannot + // complete a request Unity has nowhere to store; Send() cancels it if the call throws. + FfiClient.Instance.RegisterPendingCallback(request.RequestAsyncId, static e => e.CaptureAudioFrame, _onCaptureCallback, _onCaptureCanceled); - FfiClient.Instance.RegisterPendingCallback(requestAsyncId, static e => e.CaptureAudioFrame, Callback, OnCanceled); - try + fixed (short* pcm = frame) { + pushFrame.Buffer.DataPtr = (ulong)pcm; using var response = request.Send(); } - catch - { - var failedFrame = ReleasePendingFrameData(requestAsyncId); - if (failedFrame != null) - { - Utils.Error($"{DebugTag} request send failed asyncId={requestAsyncId} frame={failedFrame.FrameIndex} pendingAfter={PendingFrameCount()}"); - } - throw; - } } + // Main thread, posted by the FFI client. + private void OnCaptureCallback(CaptureAudioFrameCallback callback) + { + if (callback.HasError) + Utils.Error($"{DebugTag} audio capture failed asyncId={callback.AsyncId}: {callback.Error}"); + } + + // The FFI client dropped the pending callback (dispose, resume window, or a failed send). + // Debug level: at quit the client sweeps every frame whose callback Rust never sent, one + // line per frame, and a failed send is already logged as an error by the client. + private void OnCaptureCanceled() + { + Utils.Debug($"{DebugTag} capture callback canceled"); + } + + /// + /// Clears the audio processing stage's buffers. Call after the capture path restarts (e.g. a + /// microphone resume) so stale samples do not misalign the echo canceller. No-op without + /// processing. + /// + protected void ResetAudioProcessing() => _processor?.RequestReset(); + /// /// Mutes or unmutes the audio source. /// @@ -315,47 +329,12 @@ protected virtual void Dispose(bool disposing) if (disposing) Stop(); - var pendingCount = PendingFrameCount(); - if (pendingCount > 0) - Utils.Warning($"{DebugTag} dispose(disposing={disposing}) with {pendingCount} pending capture callbacks"); - - lock (_pendingFrameDataLock) - { - foreach (var pendingFrame in _pendingFrameData.Values) - { - if (pendingFrame.FrameData.IsCreated) - pendingFrame.FrameData.Dispose(); - } - _pendingFrameData.Clear(); - } + _processor?.Dispose(disposing); Handle?.Dispose(); _disposed = true; Utils.Debug($"{DebugTag} disposed"); } - private PendingAudioFrame ReleasePendingFrameData(ulong requestAsyncId) - { - PendingAudioFrame pendingFrame = null; - lock (_pendingFrameDataLock) - { - if (_pendingFrameData.TryGetValue(requestAsyncId, out pendingFrame)) - _pendingFrameData.Remove(requestAsyncId); - } - - if (pendingFrame != null && pendingFrame.FrameData.IsCreated) - pendingFrame.FrameData.Dispose(); - - return pendingFrame; - } - - private int PendingFrameCount() - { - lock (_pendingFrameDataLock) - { - return _pendingFrameData.Count; - } - } - ~RtcAudioSource() { Dispose(false); @@ -371,11 +350,6 @@ public IEnumerator PrepareAndStart() yield break; } - private static double ElapsedMilliseconds(long startedTimestamp) - { - return (Stopwatch.GetTimestamp() - startedTimestamp) * 1000.0 / Stopwatch.Frequency; - } - private string DebugTag => $"RtcAudioSource#{_debugId}"; } } diff --git a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs index b31f4dc0..fb9ed957 100644 --- a/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs +++ b/Runtime/Scripts/Internal/FFI/FfiRequestExtensions.cs @@ -152,6 +152,19 @@ public static void Inject(this FfiRequest ffiRequest, T request) case RemixAndResampleRequest remixAndResampleRequest: ffiRequest.RemixAndResample = remixAndResampleRequest; break; + // Audio processing module + case NewApmRequest newApmRequest: + ffiRequest.NewApm = newApmRequest; + break; + case ApmProcessStreamRequest apmProcessStreamRequest: + ffiRequest.ApmProcessStream = apmProcessStreamRequest; + break; + case ApmProcessReverseStreamRequest apmProcessReverseStreamRequest: + ffiRequest.ApmProcessReverseStream = apmProcessReverseStreamRequest; + break; + case ApmSetStreamDelayRequest apmSetStreamDelayRequest: + ffiRequest.ApmSetStreamDelay = apmSetStreamDelayRequest; + break; // PlatformAudio case NewPlatformAudioRequest newPlatformAudioRequest: ffiRequest.NewPlatformAudio = newPlatformAudioRequest; diff --git a/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs b/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs index 487122d1..75a0c97c 100644 --- a/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs +++ b/Runtime/Scripts/Internal/Threading/MonoBehaviourContext.cs @@ -13,6 +13,12 @@ internal class MonoBehaviourContext : MonoBehaviour private static MonoBehaviourContext _instance; private const string OBJECT_NAME = "LiveKitSDK"; + /// + /// Whether a host exists to run coroutines over time. When false, + /// drains the coroutine synchronously, which is only safe for one that finishes on its own. + /// + internal static bool CanRunCoroutines => _instance != null; + [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.BeforeSceneLoad)] private static void Init() { diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs index 1ca6c412..f1321618 100644 --- a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -47,26 +47,31 @@ public override void OnInspectorGUI() "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")); EditorGUILayout.Space(); - EditorGUILayout.LabelField("Audio Processing (PlatformAudio only)", EditorStyles.boldLabel); + EditorGUILayout.LabelField("Audio Processing", EditorStyles.boldLabel); - // Gray out audio processing options when PlatformAudio is disabled bool platformAudioEnabled = usePlatformAudio.boolValue; + EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", + "Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")); + EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", + "Enable noise suppression to remove background noise.")); + EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", + "Enable auto gain control to normalize audio levels.")); + + // Hardware processing is an ADM feature; gray it out when PlatformAudio is disabled. using (new EditorGUI.DisabledGroupScope(!platformAudioEnabled)) { - if (!platformAudioEnabled) - { - EditorGUILayout.HelpBox("Audio processing options are only available when 'Use Platform Audio' is enabled.", MessageType.Info); - } - - EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", - "Enable echo cancellation to remove echo from speaker playback.")); - EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", - "Enable noise suppression to remove background noise.")); - EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", - "Enable auto gain control to normalize audio levels.")); EditorGUILayout.PropertyField(preferHardwareProcessing, new GUIContent("Prefer Hardware Processing", - "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); + "PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have " + + "different quality characteristics.")); + } + + if (!platformAudioEnabled) + { + EditorGUILayout.HelpBox("Echo cancellation in this mode runs libwebrtc's AEC3 in the SDK; the reference is " + + "the mix on the AudioListener (a PlayoutReference component is attached automatically).", + MessageType.Info); } serializedObject.ApplyModifiedProperties(); diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 1dc27c71..5000de12 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -9,11 +9,14 @@ /// /// Manages a LiveKit room connection with local/remote audio and video tracks. /// -/// Supports two audio modes: -/// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic +/// Supports two audio modes, selected with usePlatformAudio (the Meet scene ships with +/// Unity Audio selected): +/// - PlatformAudio: Uses WebRTC's ADM for microphone capture and automatic /// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. /// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. -/// No AEC support but gives more control over audio processing. +/// Gives more control over audio processing. The same AEC/NS/AGC toggles apply: the SDK runs +/// libwebrtc's audio processing over the Microphone capture, with the mix Unity plays as the +/// echo reference (see ). /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -34,14 +37,15 @@ public class MeetManager : MonoBehaviour "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")] [SerializeField] private bool usePlatformAudio = true; - [Header("Audio Processing (PlatformAudio only)")] - [Tooltip("Enable echo cancellation to remove echo from speaker playback.")] + [Header("Audio Processing")] + [Tooltip("Enable echo cancellation. PlatformAudio: WebRTC's ADM. Unity audio: libwebrtc's AEC3 over the " + + "Microphone capture, with the mix Unity plays as the reference.")] [SerializeField] private bool echoCancellation = true; [Tooltip("Enable noise suppression to remove background noise.")] [SerializeField] private bool noiseSuppression = true; [Tooltip("Enable auto gain control to normalize audio levels.")] [SerializeField] private bool autoGainControl = true; - [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] + [Tooltip("PlatformAudio only. Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] [SerializeField] private bool preferHardwareProcessing = true; private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; @@ -605,15 +609,26 @@ private IEnumerator PublishLocalMicrophonePlatform() private IEnumerator PublishLocalMicrophoneUnity() { - Debug.Log("Publishing microphone using Unity Microphone API"); + Debug.Log($"Publishing microphone using Unity Microphone API (AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl})"); // Start the microphone here for early iOS permission request and android getting access to Microphone.devices Microphone.Start(null, true, 10, 44100); - + var audioObject = new GameObject($"My Microphone: {Microphone.devices[0]}"); audioObject.transform.SetParent(_audioTrackParent); - var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject); + // With options, MicrophoneSource runs libwebrtc's audio processing over the capture. Echo + // cancellation takes its reference from the mix Unity plays (the SDK attaches a + // PlayoutReference to the AudioListener), so it covers every remote AudioStream and the + // app's own audio. If the module cannot be created the source publishes the raw microphone. + var processing = new AudioProcessingOptions + { + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl + }; + + var rtcSource = new MicrophoneSource(Microphone.devices[0], audioObject, processing); _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, rtcSource, _room); @@ -628,6 +643,9 @@ private IEnumerator PublishLocalMicrophoneUnity() if (publish.IsError) { + // Dispose before destroying the host object so the source and its processing module + // are released now rather than by the finalizer. + rtcSource.Dispose(); Destroy(audioObject); _localAudioTrack = null; yield break; @@ -638,7 +656,9 @@ private IEnumerator PublishLocalMicrophoneUnity() _localRtcAudioSource = rtcSource; rtcSource.Start(); - Debug.Log("Microphone published via Unity Microphone API (no AEC)"); + Debug.Log(rtcSource.AudioProcessingEnabled + ? "Microphone published via Unity Microphone API (audio processing active)" + : "Microphone published via Unity Microphone API (no audio processing)"); } private void UnpublishLocalMicrophone() diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 91fc1e07..a581e401 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -902,7 +902,7 @@ MonoBehaviour: videoTrackParent: {fileID: 2128321498} participantTilePrefab: {fileID: 4315784896331113596, guid: bec493bbc3d574c07b5bbf8dd2be26b3, type: 3} frameRate: 30 - usePlatformAudio: 1 + usePlatformAudio: 0 echoCancellation: 1 noiseSuppression: 1 autoGainControl: 1 diff --git a/Tests/EditMode/AudioProcessingTests.cs b/Tests/EditMode/AudioProcessingTests.cs new file mode 100644 index 00000000..4c29ba3b --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs @@ -0,0 +1,154 @@ +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + /// + /// Pure-managed tests for the Unity-audio processing helpers: the 10 ms re-chunking, sample + /// conversion, rate rules and the delay hint. No FFI, no audio device, so they always run. + /// + public class AudioProcessingTests + { + [Test] + public void PcmRingBuffer_DrainsInWriteOrder() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2, 3, 4, 5 }, 0, 5); + + var dest = new short[3]; + Assert.IsTrue(ring.TryDrain(dest, 3)); + Assert.AreEqual(new short[] { 1, 2, 3 }, dest); + Assert.AreEqual(2, ring.Available); + + var rest = new short[2]; + Assert.IsTrue(ring.TryDrain(rest, 2)); + Assert.AreEqual(new short[] { 4, 5 }, rest); + Assert.AreEqual(0, ring.Available); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_TryDrain_WithoutEnoughSamples_ConsumesNothing() + { + var ring = new PcmRingBuffer(8); + ring.Write(new short[] { 1, 2 }, 0, 2); + + Assert.IsFalse(ring.TryDrain(new short[3], 3)); + Assert.AreEqual(2, ring.Available); + } + + [Test] + public void PcmRingBuffer_WhenFull_DropsOldest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Write(new short[] { 4, 5 }, 0, 2); + + Assert.AreEqual(1, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 2, 3, 4, 5 }, dest); + } + + [Test] + public void PcmRingBuffer_WriteLargerThanCapacity_KeepsNewest() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2 }, 0, 2); + ring.Write(new short[] { 3, 4, 5, 6, 7 }, 0, 5); + + Assert.AreEqual(3, ring.OverflowSamples); + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 4, 5, 6, 7 }, dest); + } + + [Test] + public void PcmRingBuffer_WrapsAround() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + Assert.IsTrue(ring.TryDrain(new short[2], 2)); + ring.Write(new short[] { 4, 5, 6 }, 0, 3); + + var dest = new short[4]; + Assert.IsTrue(ring.TryDrain(dest, 4)); + Assert.AreEqual(new short[] { 3, 4, 5, 6 }, dest); + Assert.AreEqual(0, ring.OverflowSamples); + } + + [Test] + public void PcmRingBuffer_Clear_Empties() + { + var ring = new PcmRingBuffer(4); + ring.Write(new short[] { 1, 2, 3 }, 0, 3); + ring.Clear(); + + Assert.AreEqual(0, ring.Available); + Assert.IsFalse(ring.TryDrain(new short[1], 1)); + } + + [Test] + public void PcmConvert_FloatToS16_ClampsAndRounds() + { + Assert.AreEqual(0, PcmConvert.FloatToS16(0f)); + Assert.AreEqual(16384, PcmConvert.FloatToS16(0.5f)); + Assert.AreEqual(-16384, PcmConvert.FloatToS16(-0.5f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(1f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-1f)); + Assert.AreEqual(short.MaxValue, PcmConvert.FloatToS16(3f)); + Assert.AreEqual(short.MinValue, PcmConvert.FloatToS16(-3f)); + } + + [Test] + public void AudioProcessingModule_SupportedApiRates_NeedWholeSampleChunks() + { + // The Rust side asserts on a frame that is not a whole multiple of 10 ms, so a rate whose + // 10 ms chunk is fractional must be refused up front. + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(48000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(44100)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(24000)); + Assert.IsTrue(AudioProcessingModule.IsSupportedApiRate(16000)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(22050)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(11025)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(0)); + Assert.IsFalse(AudioProcessingModule.IsSupportedApiRate(-48000)); + } + + [Test] + public void AudioProcessingModule_FrameSizeFor_IsTenMilliseconds() + { + Assert.AreEqual(480, AudioProcessingModule.FrameSizeFor(48000)); + Assert.AreEqual(441, AudioProcessingModule.FrameSizeFor(44100)); + Assert.AreEqual(240, AudioProcessingModule.FrameSizeFor(24000)); + Assert.IsTrue(AudioProcessingModule.IsNativeSampleRate(48000)); + Assert.IsFalse(AudioProcessingModule.IsNativeSampleRate(44100)); + } + + [Test] + public void AudioProcessingOptions_AnyProcessingEnabled_TracksTheProcessingStages() + { + Assert.IsTrue(AudioProcessingOptions.Default.AnyProcessingEnabled); + + // An all-false struct means "no processing"; PreferHardware alone is not a stage. + Assert.IsFalse(default(AudioProcessingOptions).AnyProcessingEnabled); + Assert.IsFalse(new AudioProcessingOptions { PreferHardware = true }.AnyProcessingEnabled); + Assert.IsTrue(new AudioProcessingOptions { NoiseSuppression = true }.AnyProcessingEnabled); + } + + [Test] + public void DelayHint_SumsQueueDeviceAndMicrophoneTerms() + { + // 1024 frames at 48 kHz = 21.33 ms per block; two queued blocks + 30 ms device + 50 ms + // microphone read-behind = 122.67 ms. + Assert.AreEqual(123, AudioProcessingDelayHint.EstimateMs(1024, 48000, 30)); + Assert.AreEqual(AudioProcessingDelayHint.MicrophoneReadBehindMs, AudioProcessingDelayHint.EstimateMs(0, 0, 0)); + } + + [Test] + public void DelayHint_ClampsToRange() + { + Assert.AreEqual(AudioProcessingDelayHint.MaxDelayMs, AudioProcessingDelayHint.EstimateMs(48000, 48000, 1000)); + Assert.AreEqual(AudioProcessingDelayHint.MinDelayMs, AudioProcessingDelayHint.EstimateMs(0, 0, -1000)); + } + } +} diff --git a/Tests/EditMode/AudioProcessingTests.cs.meta b/Tests/EditMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..80c9b2e9 --- /dev/null +++ b/Tests/EditMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e9ce682bd85ef402eb8ce57e87d70231 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/AudioProcessingTests.cs b/Tests/PlayMode/AudioProcessingTests.cs new file mode 100644 index 00000000..b5d5266a --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections; +using System.Runtime.InteropServices; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// Tests for the Unity-audio processing stage. They need the FFI for the module and, for the + /// echo test, Unity's audio thread. No LiveKit server. + /// + class AudioProcessingTests + { + [Test] + public void AudioProcessingModule_AcceptsTenMillisecondChunks() + { + using var apm = new AudioProcessingModule( + echoCancellerEnabled: true, + gainControllerEnabled: true, + highPassFilterEnabled: true, + noiseSuppressionEnabled: true); + + const int rate = 48000; + const int channels = 2; + var chunk = new short[AudioProcessingModule.FrameSizeFor(rate) * channels]; + var pin = GCHandle.Alloc(chunk, GCHandleType.Pinned); + try + { + var ptr = pin.AddrOfPinnedObject(); + var bytes = chunk.Length * sizeof(short); + Assert.IsNull(apm.ProcessReverseStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.ProcessStream(ptr, bytes, rate, channels)); + Assert.IsNull(apm.SetStreamDelayMs(80)); + } + finally + { + pin.Free(); + } + } + + [Test] + public void AudioProcessor_ChunksCaptureIntoTenMilliseconds() + { + var rate = AudioSettings.outputSampleRate; + const int channels = 2; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var chunkSamples = AudioProcessingModule.FrameSizeFor(rate) * channels; + var counter = new ChunkCounter(chunkSamples); + using var processor = new AudioProcessor(AudioProcessingOptions.Default, counter.OnProcessed); + processor.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + Assert.IsTrue(processor.TryProcessCapture(block, channels, rate), "block was not processed"); + + var expectedChunks = blockFrames * blocks / AudioProcessingModule.FrameSizeFor(rate); + Assert.That(counter.Chunks, Is.InRange(expectedChunks - 1, expectedChunks)); + Assert.AreEqual(0, counter.WrongSizedChunks, "a frame was not exactly one 10 ms chunk"); + + processor.Stop(); + } + + /// + /// The source-level path: options create the processor, and the capture the source reads goes + /// through it to the FFI. The Rust side takes the process down on a frame that is not 10 ms, + /// so getting through the pushes is the check; the exact chunking is covered above. + /// + [UnityTest] + public IEnumerator RtcAudioSource_WithProcessing_PublishesPushedCapture() + { + using var source = new PushAudioSource(AudioProcessingOptions.Default); + Assert.IsTrue(source.AudioProcessingEnabled, "module creation failed"); + + var rate = (int)source._expectedSampleRate; + var channels = (int)source._expectedChannels; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + source.Start(); + + const int blockFrames = 1024; + const int blocks = 10; + var block = new float[blockFrames * channels]; + for (int i = 0; i < block.Length; i++) + block[i] = 0.1f * Mathf.Sin(i * 0.05f); + for (int i = 0; i < blocks; i++) + source.Push(block, channels, rate); + + // Let the capture callbacks return before disposing. + yield return new WaitForSeconds(0.2f); + + source.Stop(); + } + + /// + /// End-to-end check of the canceller without hardware: the listener hears a noise source, + /// and a second source plays the same noise 120 ms later, probed as "microphone" and then + /// cleared so the mix contains only the far end. The capture is therefore a pure delayed + /// echo of the playout reference, which AEC3 must learn to remove. + /// + [UnityTest] + public IEnumerator AudioProcessor_CancelsDelayedEchoOfPlayout() + { + var rate = AudioSettings.outputSampleRate; + if (!AudioProcessingModule.IsSupportedApiRate(rate)) + Assert.Ignore($"output rate {rate} has no whole-sample 10 ms chunk"); + + var listenerGo = new GameObject("AecTestListener"); + listenerGo.AddComponent(); + + var clip = NoiseClip(rate, seconds: 2f, seed: 1234, amplitude: 0.3f); + + var farGo = new GameObject("AecTestFarEnd"); + var far = farGo.AddComponent(); + far.clip = clip; + far.loop = true; + + var nearGo = new GameObject("AecTestNearEnd"); + var near = nearGo.AddComponent(); + near.clip = clip; + near.loop = true; + var probe = nearGo.AddComponent(); + probe.ClearAfterInvocation(); + + var meter = new EchoMeter(); + var processor = new AudioProcessor( + new AudioProcessingOptions { EchoCancellation = true }, + meter.OnProcessed); + probe.AudioRead += (data, channels, sampleRate) => + { + meter.OnRaw(data); + processor.TryProcessCapture(data, channels, sampleRate); + }; + processor.Start(); + + var startTime = AudioSettings.dspTime + 0.2; + far.PlayScheduled(startTime); + near.PlayScheduled(startTime + 0.12); + + float rawRms, processedRms; + try + { + yield return new WaitForSeconds(1f); + if (meter.RawBlocks == 0) + Assert.Ignore("Unity's audio thread delivered no capture callbacks (no audio device?)"); + Assert.IsTrue(PlayoutReference.IsAttached, "reference not attached to the listener"); + + // Convergence time, then a clean measurement window. + yield return new WaitForSeconds(3f); + meter.ResetWindow(); + yield return new WaitForSeconds(1.5f); + + (rawRms, processedRms) = meter.Window(); + } + finally + { + processor.Dispose(); + UnityEngine.Object.Destroy(farGo); + UnityEngine.Object.Destroy(nearGo); + UnityEngine.Object.Destroy(listenerGo); + } + + Assert.Greater(rawRms, 0.01f, "near-end source produced no signal"); + + var attenuationDb = 20f * Mathf.Log10(rawRms / Mathf.Max(processedRms, 1e-6f)); + Debug.Log($"AEC3 attenuated the synthetic echo by {attenuationDb:F1} dB"); + Assert.GreaterOrEqual(attenuationDb, 6f, $"AEC3 attenuated the echo by only {attenuationDb:F1} dB"); + } + + private static AudioClip NoiseClip(int sampleRate, float seconds, int seed, float amplitude) + { + var samples = (int)(sampleRate * seconds); + var data = new float[samples]; + var random = new System.Random(seed); + for (int i = 0; i < samples; i++) + data[i] = amplitude * (float)(random.NextDouble() * 2.0 - 1.0); + + var clip = AudioClip.Create("AecTestNoise", samples, 1, sampleRate, false); + clip.SetData(data, 0); + return clip; + } + + private sealed class PushAudioSource : RtcAudioSource + { + public override event Action AudioRead; + + public PushAudioSource(AudioProcessingOptions options) + : base(RtcAudioSourceType.AudioSourceMicrophone, options) { } + + public void Push(float[] data, int channels, int sampleRate) => AudioRead?.Invoke(data, channels, sampleRate); + } + + // Counts the frames the processor hands out and checks each one is exactly one chunk. + private sealed class ChunkCounter + { + private readonly int _chunkSamples; + + public ChunkCounter(int chunkSamples) => _chunkSamples = chunkSamples; + + public int Chunks { get; private set; } + public int WrongSizedChunks { get; private set; } + + public void OnProcessed(ReadOnlySpan frame, int channels, int sampleRate) + { + if (frame.Length != _chunkSamples) WrongSizedChunks++; + Chunks++; + } + } + + // Accumulates energy of the raw near end and of the processed output. Both callbacks run on + // the Unity audio thread. + private sealed class EchoMeter + { + private readonly object _lock = new object(); + private double _rawSum; + private long _rawCount; + private double _processedSum; + private long _processedCount; + private int _rawBlocks; + + public int RawBlocks { get { lock (_lock) return _rawBlocks; } } + + public void OnRaw(float[] data) + { + double sum = 0; + for (int i = 0; i < data.Length; i++) sum += data[i] * data[i]; + lock (_lock) + { + _rawSum += sum; + _rawCount += data.Length; + _rawBlocks++; + } + } + + public void OnProcessed(ReadOnlySpan frame, int channels, int sampleRate) + { + double sum = 0; + var length = frame.Length; + for (int i = 0; i < length; i++) + { + var v = frame[i] / 32768.0; + sum += v * v; + } + lock (_lock) + { + _processedSum += sum; + _processedCount += length; + } + } + + public void ResetWindow() + { + lock (_lock) + { + _rawSum = 0; + _rawCount = 0; + _processedSum = 0; + _processedCount = 0; + } + } + + public (float raw, float processed) Window() + { + lock (_lock) return (Rms(_rawSum, _rawCount), Rms(_processedSum, _processedCount)); + } + + private static float Rms(double sum, long count) => count == 0 ? 0f : (float)Math.Sqrt(sum / count); + } + } +} diff --git a/Tests/PlayMode/AudioProcessingTests.cs.meta b/Tests/PlayMode/AudioProcessingTests.cs.meta new file mode 100644 index 00000000..4b6b1e59 --- /dev/null +++ b/Tests/PlayMode/AudioProcessingTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b22944e23e00341eb8cd913bce198b4e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/SineWaveAudioSource.cs b/Tests/PlayMode/Utils/SineWaveAudioSource.cs index 2337615b..7bfa0448 100644 --- a/Tests/PlayMode/Utils/SineWaveAudioSource.cs +++ b/Tests/PlayMode/Utils/SineWaveAudioSource.cs @@ -25,6 +25,7 @@ public sealed class SineWaveAudioSource : RtcAudioSource private double _phase; private bool _running; private bool _disposed; + private int _pumping; public SineWaveAudioSource( int channels = 2, @@ -60,6 +61,9 @@ public override void Stop() private void PumpFrame(object _) { if (!_running || _disposed) return; + // Timer callbacks can overlap on the thread pool; AudioRead must not be invoked + // concurrently, so a tick that finds the previous one still running is skipped. + if (Interlocked.Exchange(ref _pumping, 1) == 1) return; try { var buffer = new float[_samplesPerFrame * _channels]; @@ -77,6 +81,10 @@ private void PumpFrame(object _) { // Timer fires independently of FFI lifecycle; swallow errors during teardown. } + finally + { + Interlocked.Exchange(ref _pumping, 0); + } } protected override void Dispose(bool disposing)