diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 16c2b760..37964ea6 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -416,7 +416,7 @@ devices.useMicrophone(devices.microphones().get(1)); devices.useSpeaker(devices.speakers().get(0)); ``` -`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. +`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. A device that was disconnected since the list was read fails the returned future, and the device in use stays as it was. On Linux, where WebRTC's audio device module cannot tell devices apart, the default device is used whichever is given. ## Receiving Media diff --git a/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h b/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h index 65a280fa..5ad22660 100644 --- a/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h +++ b/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h @@ -46,6 +46,18 @@ namespace ffmpeg // WebRTC takes 10 ms of audio at a time and nothing else. static constexpr int kFramesPerChunk = kSampleRate / 100; + // How far a frame's timestamp may stray from where the audio + // before it ends. Containers round their timestamps, so a frame + // is followed by its sample count as long as it stays this close, + // which is also below the point where audio running ahead of + // video starts to show. + static constexpr int64_t kTimestampToleranceUs = 40000; + + // The longest stretch of missing audio filled with silence, or of + // repeated audio skipped. Beyond it the source's clock is taken + // to have jumped, as ffplay does past the same ten seconds. + static constexpr int64_t kMaxGapUs = 10000000; + AudioDecoder() = default; ~AudioDecoder(); @@ -82,6 +94,17 @@ namespace ffmpeg // Resamples one decoded frame into the pending buffer. int Resample(const AVFrame * frame); + // Holds the audio to the frame's timestamp where it strays from + // where the audio before it ends: missing audio becomes silence, + // and a frame that repeats audio already given out skips what it + // repeats, which it reports as the number of input samples to + // skip. + int FollowTimestamp(int64_t timestamp_us, int * skip); + + // Where the audio taken in so far ends, in microseconds on the + // source's timeline. + int64_t GetEndTimestampUs() const; + // Moves whatever swresample still holds into the pending buffer, // which it only hands out when told that no more input follows. int DrainResampler(); @@ -113,7 +136,9 @@ namespace ffmpeg // When the next chunk starts, in microseconds on the source's own // timeline. Chunks advance it in exact 10 ms steps, so the audio // this produces is perfectly regular whatever the container's - // packet timing looks like. + // packet timing looks like. Gaps in the source's timestamps are + // filled with silence rather than closed up, so audio stays where + // video, which follows its own timestamps, expects it. int64_t next_timestamp_us_ = 0; bool have_timestamp_ = false; diff --git a/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp index ca1a9594..8b1d5c8a 100644 --- a/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp @@ -16,6 +16,10 @@ #include "media/AudioDecoder.h" +#include +#include +#include + extern "C" { #include #include @@ -205,15 +209,6 @@ namespace ffmpeg int AudioDecoder::Resample(const AVFrame * frame) { - if (!have_timestamp_) { - int64_t pts = frame->best_effort_timestamp != AV_NOPTS_VALUE - ? frame->best_effort_timestamp : frame->pts; - - next_timestamp_us_ = pts != AV_NOPTS_VALUE - ? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0; - have_timestamp_ = true; - } - if (frame->format != in_format_ || frame->sample_rate != in_rate_ || av_channel_layout_compare(&frame->ch_layout, &in_layout_) != 0) { // The stream changed its format. What swresample still holds of @@ -232,16 +227,55 @@ namespace ffmpeg } } + int64_t pts = frame->best_effort_timestamp != AV_NOPTS_VALUE + ? frame->best_effort_timestamp : frame->pts; + int skip = 0; + + if (!have_timestamp_) { + next_timestamp_us_ = pts != AV_NOPTS_VALUE + ? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0; + have_timestamp_ = true; + } + else if (pts != AV_NOPTS_VALUE) { + int result = FollowTimestamp(av_rescale_q(pts, time_base_, AV_TIME_BASE_Q), + &skip); + + if (result < 0) { + return result; + } + } + + const int samples = frame->nb_samples - skip; + + if (samples <= 0) { + return 0; + } + // swresample holds samples back when rates differ, and those come out // of a later call, so the room needed is the delay plus this frame. int64_t delay = swr_get_delay(swr_context_, in_rate_); - int64_t capacity = av_rescale_rnd(delay + frame->nb_samples, kSampleRate, + int64_t capacity = av_rescale_rnd(delay + samples, kSampleRate, in_rate_, AV_ROUND_UP); if (capacity <= 0) { return 0; } + // swresample reads a plane for every input channel, and data has room + // for eight: planar audio of more channels has the rest only in + // extended_data. + const AVSampleFormat format = static_cast(frame->format); + const bool planar = av_sample_fmt_is_planar(format) != 0; + const int planes = planar ? frame->ch_layout.nb_channels : 1; + const int stride = av_get_bytes_per_sample(format) + * (planar ? 1 : frame->ch_layout.nb_channels); + + std::vector input(static_cast(planes)); + + for (int i = 0; i < planes; ++i) { + input[i] = frame->extended_data[i] + static_cast(skip) * stride; + } + const size_t offset = pending_.size(); pending_.resize(offset + static_cast(capacity) * channels_); @@ -249,7 +283,7 @@ namespace ffmpeg uint8_t * output = reinterpret_cast(pending_.data() + offset); int converted = swr_convert(swr_context_, &output, static_cast(capacity), - const_cast(frame->data), frame->nb_samples); + input.data(), samples); if (converted < 0) { pending_.resize(offset); @@ -263,6 +297,63 @@ namespace ffmpeg return 0; } + int AudioDecoder::FollowTimestamp(int64_t timestamp_us, int * skip) + { + const int64_t drift = timestamp_us - GetEndTimestampUs(); + + if (std::abs(drift) <= kTimestampToleranceUs) { + return 0; + } + + if (std::abs(drift) > kMaxGapUs) { + // Too far to be audio that went missing: the source's clock + // jumped, as a live stream's does when it restarts it. The audio + // goes on without a break, on the new timeline. What is pending + // is less than a chunk, so it lands just before the jump. + next_timestamp_us_ += drift; + + return 0; + } + + if (drift < 0) { + // The frame starts where audio already given out has been, so + // that much of its start goes. + *skip = static_cast(std::min( + av_rescale(-drift, in_rate_, AV_TIME_BASE), INT_MAX)); + + return 0; + } + + // Audio is missing before this frame, and silence takes its place, so + // that what follows plays at its own time and not early. What + // swresample holds back belongs before the gap. + int result = DrainResampler(); + + if (result < 0) { + return result; + } + + result = swr_init(swr_context_); + + if (result < 0) { + return result; + } + + const int64_t silence = av_rescale(drift, kSampleRate, AV_TIME_BASE); + + pending_.resize(pending_.size() + static_cast(silence) * channels_, 0); + + return 0; + } + + int64_t AudioDecoder::GetEndTimestampUs() const + { + const int64_t frames = static_cast(pending_.size() / channels_) + + swr_get_delay(swr_context_, kSampleRate); + + return next_timestamp_us_ + av_rescale(frames, AV_TIME_BASE, kSampleRate); + } + int AudioDecoder::DrainResampler() { if (swr_context_ == nullptr) { diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java index 0219e8ac..aa0fa111 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java @@ -34,6 +34,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; import dev.onvoid.webrtc.PeerConnectionFactory; import dev.onvoid.webrtc.media.audio.AudioDeviceModule; @@ -534,6 +537,74 @@ void deliversResamplerTail() throws Exception { } } + @Test + void downmixesMoreThanEightChannels() throws Exception { + // One second of 14 planar channels. A frame has the planes from the + // ninth on only in its extended data, and reading them from its data + // pointers instead crashes the process. + Path file = TestMedia.wideOpusMatroska(tempDir, 14, 50); + + try (Playback playback = new Playback(new MediaReader(file))) { + playback.player.play(); + + assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(2, playback.channels.get()); + assertTrue(Math.abs(playback.chunks.get() - 100) <= 2, + "audio chunks: " + playback.chunks.get()); + } + } + + @Test + void fillsAudioGapWithSilence() throws Exception { + // A second of audio, a second missing, and a second more. Closing the + // gap up would play the last second a second early, ahead of any + // video that follows its own timestamps. + Path file = TestMedia.pcmMatroska(tempDir, (short) 1000, + blockTimes(0, 10, 2000, 10)); + + try (Playback playback = new Playback(new MediaReader(file))) { + playback.player.play(); + + assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(300, playback.chunks.get()); + assertEquals(96000, playback.nonZeroSamples.get()); + assertEquals(IntStream.range(100, 200).boxed().collect(Collectors.toList()), + playback.silentChunks); + } + } + + @Test + void skipsRepeatedAudio() throws Exception { + // The second second starts 50 ms before the first one ends, so 50 ms + // of it repeat what has played already. + Path file = TestMedia.pcmMatroska(tempDir, (short) 1000, + blockTimes(0, 10, 950, 10)); + + try (Playback playback = new Playback(new MediaReader(file))) { + playback.player.play(); + + assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(195, playback.chunks.get()); + assertEquals(195 * 480, playback.nonZeroSamples.get()); + assertTrue(playback.silentChunks.isEmpty(), + "silent chunks: " + playback.silentChunks); + } + } + + /** + * The times of two runs of 100 ms blocks, in milliseconds. + */ + private static long[] blockTimes(long firstStart, int firstBlocks, + long secondStart, int secondBlocks) { + return LongStream.concat( + LongStream.range(0, firstBlocks).map(i -> firstStart + i * 100), + LongStream.range(0, secondBlocks).map(i -> secondStart + i * 100)) + .toArray(); + } + private static int countNonZero(byte[] data) { int count = 0; @@ -608,6 +679,7 @@ private final class Playback implements AutoCloseable { final AtomicLong firstChunkNs = new AtomicLong(); final AtomicInteger unevenChunks = new AtomicInteger(); final AtomicLong nonZeroSamples = new AtomicLong(); + final List silentChunks = Collections.synchronizedList(new ArrayList<>()); private final VideoTrackSink videoSink = frame -> { frameTimes.add(System.nanoTime()); @@ -618,7 +690,7 @@ private final class Playback implements AutoCloseable { private final AudioTrackSink audioSink = (data, bits, rate, ch, count) -> { firstChunkNs.compareAndSet(0, System.nanoTime()); - chunks.incrementAndGet(); + int index = chunks.getAndIncrement(); sampleRate.set(rate); channels.set(ch); framesPerChunk.set(count); @@ -627,7 +699,13 @@ private final class Playback implements AutoCloseable { unevenChunks.incrementAndGet(); } - nonZeroSamples.addAndGet(countNonZero(data)); + int nonZero = countNonZero(data); + + if (nonZero == 0) { + silentChunks.add(index); + } + + nonZeroSamples.addAndGet(nonZero); }; private boolean closed; diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java index 09b8c287..807b80e3 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/TestMedia.java @@ -156,6 +156,186 @@ static Path corruptFlac(Path directory, int frames, int corruptFrame) corruptFrame)); } + /** + * Writes 48 kHz, mono, 16-bit PCM in a Matroska file, in blocks of 100 ms + * of a constant, non-zero value. Matroska gives each block a time of its + * own, so the blocks need not follow each other: a time later than where + * the block before ends leaves audio missing, an earlier one repeats some. + * + * @param directory Where to write the file. + * @param value The value of every sample. + * @param blockTimes When each block starts, in milliseconds. + * + * @return The file written. + */ + static Path pcmMatroska(Path directory, short value, long... blockTimes) + throws IOException { + byte[] block = new byte[FLAC_BLOCK_SIZE * 2]; + + for (int i = 0; i < block.length; i += 2) { + block[i] = (byte) value; + block[i + 1] = (byte) (value >> 8); + } + + byte[] track = trackEntry("A_PCM/INT/LIT", null, 1, + ebmlUInt(0x6264, 16)); // BitDepth + + return Files.write(directory.resolve("pcm.mkv"), + matroska(track, blockTimes, n -> block)); + } + + /** + * Writes Opus in a Matroska file, with more channels than the eight a + * decoded frame has room for in its data pointers, in 20 ms packets of + * silence. One mono stream feeds every channel. + *

+ * The channel mapping, family 255, names no layout, so the file plays as + * the usual layout for its channel count. For 14 channels that is 9.1.4, + * whose side and top channels, all of which a stereo downmix takes in, + * are planes 8 to 11 of a decoded frame. + * + * @param directory Where to write the file. + * @param channels How many channels, 9 to 255. + * @param packets How many 20 ms packets to write. + * + * @return The file written. + */ + static Path wideOpusMatroska(Path directory, int channels, int packets) + throws IOException { + ByteBuffer head = ByteBuffer.allocate(21 + channels) + .order(ByteOrder.LITTLE_ENDIAN); + + head.put(ascii("OpusHead")) + .put((byte) 1) // Version + .put((byte) channels) + .putShort((short) 0) // Pre-skip + .putInt(48000) // Input sample rate + .putShort((short) 0) // Output gain + .put((byte) 255) // Channel mapping family + .put((byte) 1) // Streams + .put((byte) 0); // Coupled streams + + // The channel mapping follows, all zeros as allocated: every channel + // takes stream 0. + + // A CELT-only, fullband, 20 ms, mono frame (TOC 0xF8) whose range + // coded data starts with the silence flag set. + byte[] silence = { (byte) 0xF8, (byte) 0xFF, (byte) 0xFF }; + + long[] times = new long[packets]; + + for (int i = 0; i < packets; i++) { + times[i] = i * 20L; + } + + return Files.write(directory.resolve("wide-opus.mkv"), + matroska(trackEntry("A_OPUS", head.array(), channels), times, n -> silence)); + } + + /** + * A Matroska file of one audio track and one block in each cluster, so + * that every block carries its time in full. Times are in milliseconds. + */ + private static byte[] matroska(byte[] track, long[] blockTimes, + IntFunction block) { + ByteArrayOutputStream segment = new ByteArrayOutputStream(); + + // Info, with timestamps in milliseconds. + writeBytes(segment, ebml(0x1549A966, ebmlUInt(0x2AD7B1, 1000000))); + // Tracks. + writeBytes(segment, ebml(0x1654AE6B, track)); + + for (int i = 0; i < blockTimes.length; i++) { + ByteArrayOutputStream simpleBlock = new ByteArrayOutputStream(); + + // Track 1, no offset from the cluster's time, a keyframe. + simpleBlock.write(0x81); + writeInt(simpleBlock, 0, 2); + simpleBlock.write(0x80); + writeBytes(simpleBlock, block.apply(i)); + + // A cluster: its time, then the SimpleBlock. + writeBytes(segment, ebml(0x1F43B675, + ebmlUInt(0xE7, blockTimes[i]), + ebml(0xA3, simpleBlock.toByteArray()))); + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + writeBytes(out, ebml(0x1A45DFA3, + ebmlUInt(0x4286, 1), // EBMLVersion + ebmlUInt(0x42F7, 1), // EBMLReadVersion + ebmlUInt(0x42F2, 4), // EBMLMaxIDLength + ebmlUInt(0x42F3, 8), // EBMLMaxSizeLength + ebml(0x4282, ascii("matroska")), // DocType + ebmlUInt(0x4287, 4), // DocTypeVersion + ebmlUInt(0x4285, 2))); // DocTypeReadVersion + writeBytes(out, ebml(0x18538067, segment.toByteArray())); + + return out.toByteArray(); + } + + /** + * The TrackEntry of track 1, an audio track at 48 kHz, with any further + * audio settings given. + */ + private static byte[] trackEntry(String codec, byte[] codecPrivate, + int channels, byte[]... audio) { + ByteArrayOutputStream settings = new ByteArrayOutputStream(); + + writeBytes(settings, ebml(0xB5, ByteBuffer.allocate(8) + .putDouble(48000).array())); // SamplingFrequency + writeBytes(settings, ebmlUInt(0x9F, channels)); + + for (byte[] setting : audio) { + writeBytes(settings, setting); + } + + ByteArrayOutputStream entry = new ByteArrayOutputStream(); + + writeBytes(entry, ebmlUInt(0xD7, 1)); // TrackNumber + writeBytes(entry, ebmlUInt(0x73C5, 1)); // TrackUID + writeBytes(entry, ebmlUInt(0x83, 2)); // TrackType, audio + writeBytes(entry, ebml(0x86, ascii(codec))); + + if (codecPrivate != null) { + writeBytes(entry, ebml(0x63A2, codecPrivate)); + } + + writeBytes(entry, ebml(0xE1, settings.toByteArray())); + + return ebml(0xAE, entry.toByteArray()); + } + + /** + * An EBML element: its ID, which carries its own length, then its size, + * always in eight bytes, then the children or data given. + */ + private static byte[] ebml(int id, byte[]... content) { + ByteArrayOutputStream payload = new ByteArrayOutputStream(); + + for (byte[] bytes : content) { + writeBytes(payload, bytes); + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + writeInt(out, id, (32 - Integer.numberOfLeadingZeros(id) + 7) / 8); + out.write(0x01); + writeInt(out, payload.size(), 7); + writeBytes(out, payload.toByteArray()); + + return out.toByteArray(); + } + + private static byte[] ebmlUInt(int id, long value) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + writeInt(out, value, 8); + + return ebml(id, out.toByteArray()); + } + /** * A sawtooth in steps of 37, so that no two neighbouring samples are ever * equal. diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp index a2aedc57..be9101bb 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp @@ -33,6 +33,40 @@ #include "modules/audio_device/include/audio_device.h" #include "rtc_base/logging.h" +#include + +namespace +{ + // The index under which the module lists the device with the given + // descriptor, or -1 if it does not list it, as when it was unplugged + // since it was listed. Another device in its place would be one nobody + // asked for. + // + // Not every module names its devices: Linux's leave the guid empty, so + // none can be told from another there, and the module keeps to its + // default device, index 0, whichever is asked for. + int FindDevice(int16_t count, const std::string & descriptor, + const std::function & deviceName) + { + char name[webrtc::kAdmMaxDeviceNameSize]; + char guid[webrtc::kAdmMaxGuidSize]; + bool named = false; + + for (int i = 0; i < count; ++i) { + if (deviceName(static_cast(i), name, guid) != 0) { + continue; + } + if (descriptor == std::string(guid)) { + return i; + } + + named = named || guid[0] != '\0'; + } + + return (count > 0 && !named) ? 0 : -1; + } +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioDeviceModuleBase_initPlayout (JNIEnv * env, jobject caller) { @@ -184,20 +218,17 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioDeviceModuleBase_ const auto javaClass = jni::JavaClasses::get(env); const std::string devGuid = jni::JavaString::toNative(env, obj.getString(javaClass->descriptor)); - uint16_t index = 0; - int16_t deviceCount = audioModule->PlayoutDevices(); + int index = FindDevice(audioModule->PlayoutDevices(), devGuid, + [audioModule](uint16_t i, char * name, char * guid) { + return audioModule->PlayoutDeviceName(i, name, guid); + }); - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - - for (int i = 0; i < deviceCount; ++i) { - if ((audioModule->PlayoutDeviceName(i, name, guid) == 0) && devGuid == std::string(guid)) { - index = i; - break; - } + if (index < 0) { + ThrowJavaException(env, "java/lang/IllegalArgumentException", "Playout device not found"); + return; } - if (audioModule->SetPlayoutDevice(index) != 0) { + if (audioModule->SetPlayoutDevice(static_cast(index)) != 0) { env->Throw(jni::JavaError(env, "Set playout device failed")); return; } @@ -224,20 +255,17 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioDeviceModuleBase_ const auto javaClass = jni::JavaClasses::get(env); const std::string devGuid = jni::JavaString::toNative(env, obj.getString(javaClass->descriptor)); - uint16_t index = 0; - int16_t deviceCount = audioModule->RecordingDevices(); + int index = FindDevice(audioModule->RecordingDevices(), devGuid, + [audioModule](uint16_t i, char * name, char * guid) { + return audioModule->RecordingDeviceName(i, name, guid); + }); - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - - for (int i = 0; i < deviceCount; ++i) { - if ((audioModule->RecordingDeviceName(i, name, guid) == 0) && devGuid == std::string(guid)) { - index = i; - break; - } + if (index < 0) { + ThrowJavaException(env, "java/lang/IllegalArgumentException", "Recording device not found"); + return; } - if (audioModule->SetRecordingDevice(index) != 0) { + if (audioModule->SetRecordingDevice(static_cast(index)) != 0) { env->Throw(jni::JavaError(env, "Set recording device failed")); return; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java index 04de2e14..93c52e94 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java @@ -121,16 +121,11 @@ public CompletableFuture useMicrophone(AudioDevice device) { + " captures no microphone"); } - try { - audioModule.setRecordingDevice(device); - } - catch (Throwable e) { - // Most platforms switch only while not recording. - audioModule.stopRecording(); - audioModule.setRecordingDevice(device); - audioModule.initRecording(); - audioModule.startRecording(); - } + switchDevice(() -> audioModule.setRecordingDevice(device), + audioModule::stopRecording, () -> { + audioModule.initRecording(); + audioModule.startRecording(); + }); return null; }); } @@ -144,16 +139,11 @@ public CompletableFuture useSpeaker(AudioDevice device) { throw new ConfigurationException("A runtime built with AudioMode.HEADLESS plays to no speakers"); } - try { - audioModule.setPlayoutDevice(device); - } - catch (Throwable e) { - // Most platforms switch only while not playing. - audioModule.stopPlayout(); - audioModule.setPlayoutDevice(device); - audioModule.initPlayout(); - audioModule.startPlayout(); - } + switchDevice(() -> audioModule.setPlayoutDevice(device), + audioModule::stopPlayout, () -> { + audioModule.initPlayout(); + audioModule.startPlayout(); + }); return null; }); } @@ -178,6 +168,34 @@ private void changed() { }); } + /** + * Selects a device. Most platforms switch only while the module is stopped, + * so a refused switch is tried again with the module stopped, and the + * module is started again whether that works or not: a switch that fails + * leaves it on the device it had. + *

+ * A device that is no longer there fails at once. Stopping the module + * would not bring it back. + */ + static void switchDevice(Runnable select, Runnable stop, Runnable start) { + try { + select.run(); + } + catch (IllegalArgumentException e) { + throw e; + } + catch (Throwable e) { + stop.run(); + + try { + select.run(); + } + finally { + start.run(); + } + } + } + private void refresh() { cameras = read(MediaDevices::getVideoCaptureDevices); microphones = read(MediaDevices::getAudioCaptureDevices); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleBase.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleBase.java index fd6f2816..7902d6c2 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleBase.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleBase.java @@ -176,20 +176,36 @@ public void setAudioSource(AudioSource source) { public native List getRecordingDevices(); /** - * Sets the audio device to use for playback. + * Sets the audio device to use for playback. On Linux, where the module + * cannot tell its devices apart, it plays to the default device whichever + * device is given. * * @param device The audio device to use for playback. * - * @throws NullPointerException if the device is null. + * @throws NullPointerException if the device is null. + * @throws IllegalArgumentException if the device is not one of + * {@link #getPlayoutDevices()}, for example + * because it was disconnected. + * @throws Error if the module cannot switch to the + * device, which most platforms refuse + * while the module is playing. */ public native void setPlayoutDevice(AudioDevice device); /** - * Sets the audio device to use for recording. + * Sets the audio device to use for recording. On Linux, where the module + * cannot tell its devices apart, it records from the default device + * whichever device is given. * * @param device The audio device to use for recording. * - * @throws NullPointerException if the device is null. + * @throws NullPointerException if the device is null. + * @throws IllegalArgumentException if the device is not one of + * {@link #getRecordingDevices()}, for example + * because it was disconnected. + * @throws Error if the module cannot switch to the + * device, which most platforms refuse + * while the module is recording. */ public native void setRecordingDevice(AudioDevice device); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/DeviceSwitchTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/DeviceSwitchTests.java new file mode 100644 index 00000000..708d1dae --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/DeviceSwitchTests.java @@ -0,0 +1,101 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Drives {@link DevicesImpl#switchDevice} with steps that record what ran, as + * a module that refuses a switch while running, or has lost the device, would + * answer them. + */ +class DeviceSwitchTests { + + private final List steps = new ArrayList<>(); + + + @Test + void switchesWhileRunning() { + DevicesImpl.switchDevice(() -> steps.add("select"), stop(), start()); + + assertEquals(Arrays.asList("select"), steps); + } + + @Test + void retriesRefusedSwitchStopped() { + DevicesImpl.switchDevice(select(new Error("busy"), null), stop(), start()); + + assertEquals(Arrays.asList("select", "stop", "select", "start"), steps); + } + + @Test + void missingDeviceIsNotRetried() { + assertThrows(IllegalArgumentException.class, () -> DevicesImpl.switchDevice( + select(new IllegalArgumentException("gone"), null), stop(), start())); + + // Still running on the device it had. + assertEquals(Arrays.asList("select"), steps); + } + + @Test + void deviceLostBeforeRetryRestarts() { + // Unplugged between the refused switch and the second attempt. + assertThrows(IllegalArgumentException.class, () -> DevicesImpl.switchDevice( + select(new Error("busy"), new IllegalArgumentException("gone")), + stop(), start())); + + assertEquals(Arrays.asList("select", "stop", "select", "start"), steps); + } + + /** + * Selects by throwing the given errors, one per attempt, and succeeding + * once they run out or the next one is null. + */ + private Runnable select(Throwable... errors) { + int[] attempt = { 0 }; + + return () -> { + steps.add("select"); + + Throwable error = attempt[0] < errors.length ? errors[attempt[0]] : null; + + attempt[0]++; + + if (error instanceof RuntimeException) { + throw (RuntimeException) error; + } + if (error instanceof Error) { + throw (Error) error; + } + }; + } + + private Runnable stop() { + return () -> steps.add("stop"); + } + + private Runnable start() { + return () -> steps.add("start"); + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleTest.java index 4f729590..e104b67e 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/AudioDeviceModuleTest.java @@ -1,5 +1,7 @@ package dev.onvoid.webrtc.media.audio; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.junit.jupiter.api.*; class AudioDeviceModuleTest { @@ -45,4 +47,22 @@ public int onPlaybackData(byte[] audioSamples, int nSamples, module.setAudioSource(source); } + + @Test + void missingPlayoutDeviceIsRejected() { + // Not another device in its place, as the first one listed. + assertThrows(IllegalArgumentException.class, + () -> module.setPlayoutDevice(missingDevice())); + } + + @Test + void missingRecordingDeviceIsRejected() { + assertThrows(IllegalArgumentException.class, + () -> module.setRecordingDevice(missingDevice())); + } + + /** A device as one unplugged since it was listed looks. */ + private static AudioDevice missingDevice() { + return new AudioDevice("Unplugged", "no-such-device") { }; + } }