diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 495ff157..86025a8c 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -277,7 +277,7 @@ class MyDecoder implements VideoDecoder { - The image and its buffer are read-only and valid only until `decode()` returns; copy the data to keep it longer. - A decoded frame must carry the capture time of the image it was decoded from as its timestamp. -- `onDecodedFrame()` takes a reference of its own, so the caller still releases the frame it created. Frame buffers other than `NativeI420Buffer` are copied, and have to provide direct byte buffers. +- `onDecodedFrame()` takes a reference of its own, so the caller still releases the frame it created. A buffer whose `toI420()` returns a `NativeI420Buffer` is shared; any other is copied, and has to provide direct byte buffers as large as its strides and height require. ## Errors diff --git a/docs/guide/video/custom-video-source.md b/docs/guide/video/custom-video-source.md index 0b3da97e..8ec5f805 100644 --- a/docs/guide/video/custom-video-source.md +++ b/docs/guide/video/custom-video-source.md @@ -68,9 +68,11 @@ VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); videoSource.pushFrame(frame); // Don't forget to release the frame when done -frame.dispose(); +frame.release(); ``` +A frame can also carry a `VideoFrameBuffer` of your own. The source takes its pixels from `toI420()`: a `NativeI420Buffer` returned there is used as it is, and any other `I420Buffer` is copied, so its planes have to be direct byte buffers that are as large as their strides and the height require. A buffer that does not fit its description is rejected with a `RuntimeException`. `VideoFrame.copy()` works the same way, and always returns a frame with a `NativeI420Buffer`. + ## Video Format Considerations When pushing video frames, you need to consider the following: diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeI420Buffer.h b/webrtc-jni/src/main/cpp/include/JNI_NativeI420Buffer.h index 09f2303d..09b174f7 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_NativeI420Buffer.h +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeI420Buffer.h @@ -18,10 +18,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_video_NativeI420Buffer * Method: copy - * Signature: (IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;I)Ldev/onvoid/webrtc/media/video/NativeI420Buffer; + * Signature: (Ldev/onvoid/webrtc/media/video/VideoFrameBuffer;)Ldev/onvoid/webrtc/media/video/NativeI420Buffer; */ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_video_NativeI420Buffer_copy - (JNIEnv*, jclass, jint, jint, jobject, jint, jobject, jint, jobject, jint); + (JNIEnv *, jclass, jobject); /* * Class: dev_onvoid_webrtc_media_video_NativeI420Buffer diff --git a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h index 6b79e255..ce5205a8 100644 --- a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h +++ b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h @@ -37,9 +37,12 @@ namespace jni // is released. Returns null if the pixels cannot be converted. JavaLocalRef toJava(JNIEnv * env, const webrtc::VideoFrame & frame); - // Returns the native pixel buffer of a Java VideoFrameBuffer. A - // NativeI420Buffer is shared; any other buffer is converted to I420 - // in Java and copied. Returns null if it has no readable pixels. + // Returns the native pixel buffer of a Java VideoFrameBuffer, which + // is converted to I420 in Java first. A NativeI420Buffer that comes + // out of that is shared; any other I420 buffer is copied. Returns + // null if it has no readable pixels, such as a released buffer or + // one with heap byte buffers, and throws a RuntimeException if its + // size, strides and planes do not fit together. webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer); } diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeI420Buffer.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeI420Buffer.cpp index 2d03b04c..d14f8073 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_NativeI420Buffer.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeI420Buffer.cpp @@ -42,16 +42,19 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_video_NativeI420Buffer_al } JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_video_NativeI420Buffer_copy -(JNIEnv* env, jclass caller, jint width, jint height, jobject jSrcY, jint srcStrideY, - jobject jSrcU, jint srcStrideU, jobject jSrcV, jint srcStrideV) +(JNIEnv * env, jclass caller, jobject jSource) { try { - const uint8_t * src_y = static_cast(env->GetDirectBufferAddress(jSrcY)); - const uint8_t * src_u = static_cast(env->GetDirectBufferAddress(jSrcU)); - const uint8_t * src_v = static_cast(env->GetDirectBufferAddress(jSrcV)); + webrtc::scoped_refptr source = + jni::VideoFrame::toNativeBuffer(env, jni::JavaLocalRef(env, jSource)); + webrtc::scoped_refptr sourceI420 = source ? source->ToI420() : nullptr; - webrtc::scoped_refptr i420Buffer = webrtc::I420Buffer::Copy(width, height, - src_y, srcStrideY, src_u, srcStrideU, src_v, srcStrideV); + if (sourceI420 == nullptr) { + env->Throw(jni::JavaRuntimeException(env, "The buffer cannot be read")); + return nullptr; + } + + webrtc::scoped_refptr i420Buffer = webrtc::I420Buffer::Copy(*sourceI420); jni::JavaLocalRef jBuffer = jni::I420Buffer::toJava(env, i420Buffer); diff --git a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp index 98b4f92a..e5e181e3 100644 --- a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp +++ b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp @@ -61,43 +61,14 @@ namespace jni int rotation = obj.getInt(javaClass->rotation); int64_t timestamp_ns = obj.getLong(javaClass->timestampNs); JavaLocalRef jBuffer = obj.getObject(javaClass->buffer); - - /* - // Get the I420Buffer properties - const auto i420Class = JavaClasses::get(env); - JavaObject bufferObj(env, jBuffer); - - int width = bufferObj.getInt(i420Class->width); - int height = bufferObj.getInt(i420Class->height); - - JavaLocalRef dataY = bufferObj.getObject(i420Class->dataY); - JavaLocalRef dataU = bufferObj.getObject(i420Class->dataU); - JavaLocalRef dataV = bufferObj.getObject(i420Class->dataV); - - int strideY = bufferObj.getInt(i420Class->strideY); - int strideU = bufferObj.getInt(i420Class->strideU); - int strideV = bufferObj.getInt(i420Class->strideV); - - // Get the direct buffer addresses - const uint8_t * y_data = static_cast(env->GetDirectBufferAddress(dataY.get())); - const uint8_t * u_data = static_cast(env->GetDirectBufferAddress(dataU.get())); - const uint8_t * v_data = static_cast(env->GetDirectBufferAddress(dataV.get())); - - // Create a native I420Buffer - webrtc::scoped_refptr buffer = webrtc::I420Buffer::Copy( - width, height, y_data, strideY, u_data, strideU, v_data, strideV); - */ - - webrtc::I420BufferInterface * i420Buffer = GetHandle(env, jBuffer); - - // A buffer that has been released for good has no pixels left. - if (i420Buffer == nullptr) { + + webrtc::scoped_refptr buffer = toNativeBuffer(env, jBuffer); + + if (buffer == nullptr) { throw JavaWrappedException(JavaLocalRef(env, - JavaRuntimeException(env, "The buffer of the VideoFrame has been released"))); + JavaRuntimeException(env, "The buffer of the VideoFrame cannot be read"))); } - webrtc::scoped_refptr buffer(i420Buffer); - return webrtc::VideoFrame::Builder() .set_video_frame_buffer(buffer) //.set_timestamp_rtp(timestamp_rtp) @@ -136,21 +107,13 @@ namespace jni webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer) { - const auto nativeClass = JavaClasses::get(env); - - if (env->IsInstanceOf(javaBuffer, nativeClass->cls)) { - webrtc::I420BufferInterface * buffer = GetHandle(env, javaBuffer); - - // Released for good, so there is nothing to read. - if (buffer == nullptr) { - return nullptr; - } - - return webrtc::scoped_refptr(buffer); + if (javaBuffer.get() == nullptr) { + return nullptr; } const auto javaClass = JavaClasses::get(env); + // A NativeI420Buffer returns itself, and so may any other buffer that keeps its pixels in one. JavaLocalRef i420(env, env->CallObjectMethod(javaBuffer, javaClass->toI420)); ExceptionCheck(env); @@ -158,6 +121,19 @@ namespace jni return nullptr; } + const auto nativeClass = JavaClasses::get(env); + + if (env->IsInstanceOf(i420, nativeClass->cls)) { + webrtc::I420BufferInterface * buffer = GetHandle(env, i420); + + // Released for good, so there is nothing to read. + if (buffer == nullptr) { + return nullptr; + } + + return webrtc::scoped_refptr(buffer); + } + const jint width = env->CallIntMethod(i420, javaClass->getWidth); const jint height = env->CallIntMethod(i420, javaClass->getHeight); const jint strideY = env->CallIntMethod(i420, javaClass->getStrideY); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/NativeI420Buffer.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/NativeI420Buffer.java index 73c4c8c8..092c92f0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/NativeI420Buffer.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/NativeI420Buffer.java @@ -114,9 +114,11 @@ public String toString() { * Creates a copy of this I420 buffer. This method performs a deep copy of all data in the buffer. * * @return A new NativeI420Buffer instance with the same dimensions and data as this buffer. + * + * @throws RuntimeException if this buffer has been released. */ public NativeI420Buffer copy() { - return copy(width, height, dataY.slice(), strideY, dataU.slice(), strideU, dataV.slice(), strideV); + return copy(this); } /** @@ -148,10 +150,14 @@ public static NativeI420Buffer allocate(int width, int height) { private static native NativeI420Buffer allocateBuffer(int width, int height); /** - * Creates a copy of an I420 buffer with the given dimensions and data. + * Copies the pixels of any buffer, as {@link VideoFrameBuffer#toI420()} provides them, into a new + * buffer. + * + * @throws RuntimeException if the buffer has no readable pixels, such as a released buffer or one + * with heap byte buffers, or if the size, strides and planes of the I420 + * buffer do not fit together. */ - private static native NativeI420Buffer copy(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU, - int strideU, ByteBuffer dataV, int strideV); + static native NativeI420Buffer copy(VideoFrameBuffer buffer); /** * Crops this buffer without copying. The cropped buffer shares the pixels of this one and holds a diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoFrame.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoFrame.java index 79ec2ead..248a9d4b 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoFrame.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoFrame.java @@ -74,14 +74,18 @@ public VideoFrame(VideoFrameBuffer buffer, int rotation, long timestampNs) { } /** - * Creates a deep copy of this VideoFrame. The new frame will have its own copy of the buffer data. + * Creates a deep copy of this VideoFrame. The new frame will have its own copy of the buffer data, + * in a {@link NativeI420Buffer} made from what {@link VideoFrameBuffer#toI420()} of the buffer + * returns. * * @return A new VideoFrame with a copy of the buffer data. + * + * @throws RuntimeException if the buffer has no readable pixels, such as a released buffer or one + * with heap byte buffers, or if the size, strides and planes of the I420 + * buffer do not fit together. */ public VideoFrame copy() { - NativeI420Buffer nativeBuffer = (NativeI420Buffer) buffer; - - return new VideoFrame(nativeBuffer.copy(), rotation, timestampNs); + return new VideoFrame(NativeI420Buffer.copy(buffer), rotation, timestampNs); } @Override diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomFrameBufferTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomFrameBufferTest.java new file mode 100644 index 00000000..8bb1b843 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomFrameBufferTest.java @@ -0,0 +1,348 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import dev.onvoid.webrtc.TestBase; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Buffers of the application's own, which provide their pixels through {@link VideoFrameBuffer#toI420()}. + */ +class CustomFrameBufferTest extends TestBase { + + private static final int WIDTH = 6; + + private static final int HEIGHT = 4; + + private CustomVideoSource videoSource; + + private VideoTrack videoTrack; + + /** The luma of every frame that reached the track, row by row. */ + private final List received = new ArrayList<>(); + + private final VideoTrackSink sink = frame -> received.add(lumaOf(frame.buffer.toI420())); + + + @BeforeEach + void init() { + // The instance, and so the list, lives as long as the class. + received.clear(); + + videoSource = new CustomVideoSource(); + videoTrack = factory.createVideoTrack("videoTrack", videoSource); + videoTrack.addSink(sink); + } + + @AfterEach + void dispose() { + videoTrack.removeSink(sink); + videoTrack.dispose(); + videoSource.dispose(); + } + + @Test + void copyWrappedNativeBuffer() { + NativeI420Buffer pixels = NativeI420Buffer.allocate(WIDTH, HEIGHT); + fillLuma(pixels); + + VideoFrame frame = new VideoFrame(new WrappingBuffer(pixels), 0); + VideoFrame copy = frame.copy(); + + assertNotSame(pixels, copy.buffer); + assertArrayEquals(expectedLuma(), lumaOf(copy.buffer.toI420())); + + // The copy has pixels of its own. + frame.release(); + + assertArrayEquals(expectedLuma(), lumaOf(copy.buffer.toI420())); + + copy.release(); + } + + @Test + void copyCustomI420Buffer() { + VideoFrame frame = new VideoFrame(new DirectI420Buffer(WIDTH + 3, 0), 0); + VideoFrame copy = frame.copy(); + + assertArrayEquals(expectedLuma(), lumaOf(copy.buffer.toI420())); + + copy.release(); + } + + @Test + void copyReleasedBuffer() { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + buffer.release(); + + assertThrows(RuntimeException.class, buffer::copy); + assertThrows(RuntimeException.class, () -> new VideoFrame(buffer, 0).copy()); + } + + @Test + void pushWrappedNativeBuffer() { + NativeI420Buffer pixels = NativeI420Buffer.allocate(WIDTH, HEIGHT); + fillLuma(pixels); + + VideoFrame frame = new VideoFrame(new WrappingBuffer(pixels), 0); + videoSource.pushFrame(frame); + frame.release(); + + assertReceived(); + } + + @Test + void pushCustomI420Buffer() { + // Padded rows, as many capture APIs deliver them. + VideoFrame frame = new VideoFrame(new DirectI420Buffer(WIDTH + 3, 0), 0); + videoSource.pushFrame(frame); + + assertReceived(); + } + + @Test + void pushBufferWithShortPlanes() { + // Each plane lacks the last byte that its strides and height require. + VideoFrame frame = new VideoFrame(new DirectI420Buffer(WIDTH, -1), 0); + + assertThrows(RuntimeException.class, () -> videoSource.pushFrame(frame)); + assertThrows(RuntimeException.class, frame::copy); + assertEquals(0, received.size()); + } + + @Test + void pushBufferWithShortStride() { + VideoFrame frame = new VideoFrame(new DirectI420Buffer(WIDTH - 1, 0), 0); + + assertThrows(RuntimeException.class, () -> videoSource.pushFrame(frame)); + assertEquals(0, received.size()); + } + + @Test + void pushHeapBuffer() { + DirectI420Buffer buffer = new DirectI420Buffer(WIDTH, 0) { + + @Override + public ByteBuffer getDataY() { + return ByteBuffer.allocate(WIDTH * HEIGHT); + } + }; + VideoFrame frame = new VideoFrame(buffer, 0); + + assertThrows(RuntimeException.class, () -> videoSource.pushFrame(frame)); + assertEquals(0, received.size()); + } + + private void assertReceived() { + assertEquals(1, received.size()); + assertArrayEquals(expectedLuma(), received.get(0)); + } + + private static void fillLuma(I420Buffer buffer) { + ByteBuffer y = buffer.getDataY(); + + for (int row = 0; row < buffer.getHeight(); row++) { + for (int col = 0; col < buffer.getWidth(); col++) { + y.put(row * buffer.getStrideY() + col, lumaValue(row, col)); + } + } + } + + private static byte[] lumaOf(I420Buffer buffer) { + ByteBuffer y = buffer.getDataY(); + byte[] luma = new byte[buffer.getWidth() * buffer.getHeight()]; + + for (int row = 0; row < buffer.getHeight(); row++) { + for (int col = 0; col < buffer.getWidth(); col++) { + luma[row * buffer.getWidth() + col] = y.get(row * buffer.getStrideY() + col); + } + } + + return luma; + } + + private static byte[] expectedLuma() { + byte[] luma = new byte[WIDTH * HEIGHT]; + + for (int row = 0; row < HEIGHT; row++) { + for (int col = 0; col < WIDTH; col++) { + luma[row * WIDTH + col] = lumaValue(row, col); + } + } + + return luma; + } + + private static byte lumaValue(int row, int col) { + return (byte) (16 + row * 16 + col); + } + + + + /** + * A buffer that is not an I420Buffer itself and keeps its pixels in a NativeI420Buffer. + */ + private static class WrappingBuffer implements VideoFrameBuffer { + + private final NativeI420Buffer pixels; + + + WrappingBuffer(NativeI420Buffer pixels) { + this.pixels = pixels; + } + + @Override + public int getWidth() { + return pixels.getWidth(); + } + + @Override + public int getHeight() { + return pixels.getHeight(); + } + + @Override + public I420Buffer toI420() { + return pixels; + } + + @Override + public VideoFrameBuffer cropAndScale(int cropX, int cropY, int cropWidth, + int cropHeight, int scaleWidth, int scaleHeight) { + return pixels.cropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + } + + @Override + public void retain() { + pixels.retain(); + } + + @Override + public void release() { + pixels.release(); + } + } + + + + /** + * An I420 buffer in direct byte buffers of its own, with a luma stride of choice. + */ + private static class DirectI420Buffer implements I420Buffer { + + private final int strideY; + + private final int strideUV; + + private final ByteBuffer dataY; + + private final ByteBuffer dataU; + + private final ByteBuffer dataV; + + + /** + * @param strideY The stride of the luma plane, which may be shorter than the width. + * @param extraBytes Added to the size that each plane needs, so a negative number makes it too short. + */ + DirectI420Buffer(int strideY, int extraBytes) { + this.strideY = strideY; + this.strideUV = (WIDTH + 1) / 2; + + int chromaHeight = (HEIGHT + 1) / 2; + + dataY = ByteBuffer.allocateDirect(Math.max(strideY, WIDTH) * (HEIGHT - 1) + WIDTH + extraBytes); + dataU = ByteBuffer.allocateDirect(strideUV * chromaHeight + extraBytes); + dataV = ByteBuffer.allocateDirect(strideUV * chromaHeight + extraBytes); + + if (strideY >= WIDTH && extraBytes >= 0) { + fillLuma(this); + } + } + + @Override + public ByteBuffer getDataY() { + return dataY.slice(); + } + + @Override + public ByteBuffer getDataU() { + return dataU.slice(); + } + + @Override + public ByteBuffer getDataV() { + return dataV.slice(); + } + + @Override + public int getStrideY() { + return strideY; + } + + @Override + public int getStrideU() { + return strideUV; + } + + @Override + public int getStrideV() { + return strideUV; + } + + @Override + public int getWidth() { + return WIDTH; + } + + @Override + public int getHeight() { + return HEIGHT; + } + + @Override + public I420Buffer toI420() { + return this; + } + + @Override + public VideoFrameBuffer cropAndScale(int cropX, int cropY, int cropWidth, + int cropHeight, int scaleWidth, int scaleHeight) { + throw new UnsupportedOperationException(); + } + + @Override + public void retain() { + } + + @Override + public void release() { + } + } +}