From 2819e24a154f9198352e201a067bbb81cfb0e8e0 Mon Sep 17 00:00:00 2001 From: Brandon Toner Date: Wed, 30 Sep 2026 15:59:57 +0000 Subject: [PATCH] feat: Add java.nio.ByteBuffer support to IonSystem Add ByteBuffer overloads to the three most common IonSystem entry points so callers holding NIO buffers no longer need to copy into a byte[] themselves: - IonSystem.iterate(ByteBuffer) - IonSystem.newReader(ByteBuffer) - IonSystem.singleValue(ByteBuffer) The methods are declared as interface default methods that delegate to the existing byte[] overloads, so the addition is source- and binary-compatible for external IonSystem implementors. A shared helper, _Private_ByteBufferUtils.toByteArrayConsuming, copies the buffer's readable region ([position, limit)) via a relative bulk get, which works for array-backed, direct (off-heap), and read-only buffers alike and advances position to limit (consume semantics). The reader/iterator operates over a private copy, so later buffer mutations do not affect reading. Add six FROM_BYTE_BUFFER_* constants to the ReaderMaker test enum (binary, text, direct, read-only, and sliced sub-range variants) so the shared reader suites exercise the new path, plus IonSystemByteBufferTest covering all buffer kinds, both formats, the position/limit contract, snapshot isolation, and null/EOF/multi-value error behavior. --- src/main/java/com/amazon/ion/IonSystem.java | 87 +++++ .../ion/impl/_Private_ByteBufferUtils.java | 56 ++++ .../amazon/ion/IonSystemByteBufferTest.java | 306 ++++++++++++++++++ src/test/java/com/amazon/ion/ReaderMaker.java | 151 +++++++++ 4 files changed, 600 insertions(+) create mode 100644 src/main/java/com/amazon/ion/impl/_Private_ByteBufferUtils.java create mode 100644 src/test/java/com/amazon/ion/IonSystemByteBufferTest.java diff --git a/src/main/java/com/amazon/ion/IonSystem.java b/src/main/java/com/amazon/ion/IonSystem.java index aa1b4b93f5..e08e3073c4 100644 --- a/src/main/java/com/amazon/ion/IonSystem.java +++ b/src/main/java/com/amazon/ion/IonSystem.java @@ -15,12 +15,14 @@ package com.amazon.ion; +import com.amazon.ion.impl._Private_ByteBufferUtils; import com.amazon.ion.system.IonSystemBuilder; import com.amazon.ion.system.IonTextWriterBuilder; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.nio.ByteBuffer; import java.util.Date; import java.util.Iterator; @@ -319,6 +321,36 @@ public SymbolTable newSharedSymbolTable(String name, @Deprecated public Iterator iterate(byte[] ionData); + /** + * Creates an iterator over the Ion data in the given {@link ByteBuffer}. + * Values returned by the iterator have no container. + *

+ * The iterator will automatically consume Ion system IDs and local symbol + * tables; they will not be returned by the iterator. + *

+ * This method reads the buffer's remaining bytes (from its current + * {@code position} up to its {@code limit}) and will auto-detect and + * uncompress GZIPped Ion data. It works for all kinds of {@link ByteBuffer}, + * including array-backed, direct (off-heap), and read-only buffers. + *

+ * On return, the buffer's {@code position} has been advanced to its + * {@code limit} (its remaining bytes have been consumed); the buffer's + * {@code limit}, {@code capacity}, and contents are otherwise unchanged. + * The returned iterator operates over a private copy of the bytes, so later + * modifications to the buffer do not affect iteration. + * + * @param ionData may be either Ion binary data, or (UTF-8) Ion text, or + * GZIPped Ion data. Must not be null. + * + * @return a new iterator instance. + * + * @throws NullPointerException if {@code ionData} is null. + */ + default Iterator iterate(ByteBuffer ionData) + { + return iterate(_Private_ByteBufferUtils.toByteArrayConsuming(ionData)); + } + /** *

* Creates an iterator over Ion data. @@ -394,6 +426,34 @@ public SymbolTable newSharedSymbolTable(String name, */ public IonValue singleValue(byte[] ionData, int offset, int len); + /** + * Extracts a single value from the Ion data in the given {@link ByteBuffer}. + *

+ * This method reads the buffer's remaining bytes (from its current + * {@code position} up to its {@code limit}) and will auto-detect and + * uncompress GZIPped Ion data. It works for all kinds of {@link ByteBuffer}, + * including array-backed, direct (off-heap), and read-only buffers. + *

+ * On return, the buffer's {@code position} has been advanced to its + * {@code limit} (its remaining bytes have been consumed); the buffer's + * {@code limit}, {@code capacity}, and contents are otherwise unchanged. + * + * @param ionData may be either Ion binary data, or (UTF-8) Ion text, or + * GZIPped Ion data. Must not be null. + * + * @return the first (and only) user value in the data; not null. + * + * @throws NullPointerException if {@code ionData} is null. + * @throws UnexpectedEofException if the data doesn't contain any user + * values. + * @throws IonException if the data does not contain exactly one user + * value. + */ + default IonValue singleValue(ByteBuffer ionData) + { + return singleValue(_Private_ByteBufferUtils.toByteArrayConsuming(ionData)); + } + //------------------------------------------------------------------------- // IonReader creation @@ -448,6 +508,33 @@ public SymbolTable newSharedSymbolTable(String name, */ public IonReader newReader(byte[] ionData, int offset, int len); + /** + * Creates a new {@link IonReader} instance over the Ion data in the given + * {@link ByteBuffer}, detecting whether it's text or binary data. + *

+ * This method reads the buffer's remaining bytes (from its current + * {@code position} up to its {@code limit}) and will auto-detect and + * uncompress GZIPped Ion data. It works for all kinds of {@link ByteBuffer}, + * including array-backed, direct (off-heap), and read-only buffers. + *

+ * On return, the buffer's {@code position} has been advanced to its + * {@code limit} (its remaining bytes have been consumed); the buffer's + * {@code limit}, {@code capacity}, and contents are otherwise unchanged. + * The returned reader operates over a private copy of the bytes, so later + * modifications to the buffer do not affect reading. + * + * @param ionData may be either Ion binary data, or (UTF-8) Ion text, or + * GZIPped Ion data. Must not be null. + * + * @return a new reader instance. + * + * @throws NullPointerException if {@code ionData} is null. + */ + default IonReader newReader(ByteBuffer ionData) + { + return newReader(_Private_ByteBufferUtils.toByteArrayConsuming(ionData)); + } + /** * Creates a new {@link IonReader} instance over a stream of Ion data, * detecting whether it's text or binary data. diff --git a/src/main/java/com/amazon/ion/impl/_Private_ByteBufferUtils.java b/src/main/java/com/amazon/ion/impl/_Private_ByteBufferUtils.java new file mode 100644 index 0000000000..9c92fb3771 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/_Private_ByteBufferUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright 2007-2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import java.nio.ByteBuffer; + +/** + * Internal helpers for adapting {@link java.nio.ByteBuffer} inputs to the + * {@code byte[]}-based factory methods on {@link com.amazon.ion.IonSystem}. + *

+ * This is an internal API and is subject to change without notice. + */ +public final class _Private_ByteBufferUtils +{ + private _Private_ByteBufferUtils() {} + + /** + * Copies the readable region of the given buffer (the bytes between its + * current {@code position} and its {@code limit}) into a newly allocated + * {@code byte[]}, advancing the buffer's {@code position} to its + * {@code limit} as a side effect (i.e. the remaining bytes are consumed). + *

+ * This uses a relative bulk {@link ByteBuffer#get(byte[])}, which works for + * every kind of {@link ByteBuffer} — array-backed, direct (off-heap), + * and read-only — and never throws {@link java.nio.ReadOnlyBufferException}. + * The returned array is an independent copy, so subsequent mutations of the + * buffer do not affect it. + * + * @param buffer the buffer to read from; must not be null. + * @return a new array containing the buffer's (former) remaining bytes. + * @throws NullPointerException if {@code buffer} is null. + */ + public static byte[] toByteArrayConsuming(ByteBuffer buffer) + { + if (buffer == null) + { + throw new NullPointerException("ionData"); + } + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + return bytes; + } +} diff --git a/src/test/java/com/amazon/ion/IonSystemByteBufferTest.java b/src/test/java/com/amazon/ion/IonSystemByteBufferTest.java new file mode 100644 index 0000000000..2f1007d9cf --- /dev/null +++ b/src/test/java/com/amazon/ion/IonSystemByteBufferTest.java @@ -0,0 +1,306 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import com.amazon.ion.impl._Private_Utils; +import java.nio.ByteBuffer; +import java.util.Iterator; +import org.junit.Test; + +/** + * Unit tests for the {@link java.nio.ByteBuffer} overloads on {@link IonSystem}: + * {@link IonSystem#iterate(ByteBuffer)}, {@link IonSystem#newReader(ByteBuffer)}, + * and {@link IonSystem#singleValue(ByteBuffer)}. + *

+ * The tests exercise all supported buffer kinds (array-backed writable, direct + * off-heap, read-only, and a sliced sub-range of a larger backing array) for + * both binary and (UTF-8) text Ion, and verify the documented buffer + * position/limit contract, snapshot isolation, and error behavior. + */ +public class IonSystemByteBufferTest + extends IonTestCase +{ + // Three top-level values, used for the iterate/multi-value cases. + private static final String THREE_VALUES = "1 2 3"; + // A single value, used for the singleValue/newReader cases. + private static final String ONE_VALUE = "\"hello\""; + + /** How the ByteBuffer under test is constructed from a byte[]. */ + private enum BufferKind + { + /** {@link ByteBuffer#wrap(byte[])}: array-backed, writable. */ + ARRAY_BACKED + { + @Override + ByteBuffer wrap(byte[] data) + { + return ByteBuffer.wrap(data); + } + }, + /** {@link ByteBuffer#allocateDirect(int)}: off-heap, hasArray()==false. */ + DIRECT + { + @Override + ByteBuffer wrap(byte[] data) + { + ByteBuffer b = ByteBuffer.allocateDirect(data.length); + b.put(data); + b.flip(); + return b; + } + }, + /** {@link ByteBuffer#asReadOnlyBuffer()}: isReadOnly()==true. */ + READ_ONLY + { + @Override + ByteBuffer wrap(byte[] data) + { + return ByteBuffer.wrap(data).asReadOnlyBuffer(); + } + }, + /** + * A {@link ByteBuffer#slice()} whose readable region is exactly + * {@code data}, but which sits in the middle of a larger backing array + * (non-zero {@code arrayOffset}). Bytes outside the readable region are + * filled with a sentinel to catch over-reads. + */ + SLICED_SUBRANGE + { + @Override + ByteBuffer wrap(byte[] data) + { + int prefix = 13; + int suffix = 17; + byte[] padded = new byte[prefix + data.length + suffix]; + // Sentinel bytes that must never be read. + java.util.Arrays.fill(padded, (byte) 0x7F); + System.arraycopy(data, 0, padded, prefix, data.length); + ByteBuffer backing = ByteBuffer.wrap(padded); + backing.position(prefix); + backing.limit(prefix + data.length); + return backing.slice(); + } + }; + + abstract ByteBuffer wrap(byte[] data); + } + + private byte[] binary(String ionText) + { + return encode(ionText); + } + + private byte[] text(String ionText) + { + return _Private_Utils.utf8(ionText); + } + + //======================================================================== + // iterate(ByteBuffer) + + @Test + public void testIterateBinaryAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(binary(THREE_VALUES)); + Iterator it = system().iterate(buffer); + assertThreeInts(it, kind.name()); + } + } + + @Test + public void testIterateTextAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(text(THREE_VALUES)); + Iterator it = system().iterate(buffer); + assertThreeInts(it, kind.name()); + } + } + + @Test + public void testIterateEmptyBuffer() + { + Iterator it = system().iterate(ByteBuffer.wrap(new byte[0])); + assertFalse(it.hasNext()); + } + + @Test(expected = NullPointerException.class) + public void testIterateNullBuffer() + { + system().iterate((ByteBuffer) null); + } + + //======================================================================== + // newReader(ByteBuffer) + + @Test + public void testNewReaderBinaryAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(binary(ONE_VALUE)); + IonReader reader = system().newReader(buffer); + assertSame("kind=" + kind, IonType.STRING, reader.next()); + assertEquals("kind=" + kind, "hello", reader.stringValue()); + assertNull("kind=" + kind, reader.next()); + } + } + + @Test + public void testNewReaderTextAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(text(ONE_VALUE)); + IonReader reader = system().newReader(buffer); + assertSame("kind=" + kind, IonType.STRING, reader.next()); + assertEquals("kind=" + kind, "hello", reader.stringValue()); + assertNull("kind=" + kind, reader.next()); + } + } + + @Test(expected = NullPointerException.class) + public void testNewReaderNullBuffer() + { + system().newReader((ByteBuffer) null); + } + + //======================================================================== + // singleValue(ByteBuffer) + + @Test + public void testSingleValueBinaryAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(binary(ONE_VALUE)); + IonValue v = system().singleValue(buffer); + assertEquals("kind=" + kind, "hello", ((IonString) v).stringValue()); + } + } + + @Test + public void testSingleValueTextAllKinds() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(text(ONE_VALUE)); + IonValue v = system().singleValue(buffer); + assertEquals("kind=" + kind, "hello", ((IonString) v).stringValue()); + } + } + + @Test(expected = UnexpectedEofException.class) + public void testSingleValueEmptyBuffer() + { + system().singleValue(ByteBuffer.wrap(new byte[0])); + } + + @Test(expected = IonException.class) + public void testSingleValueMultipleValues() + { + system().singleValue(ByteBuffer.wrap(text(THREE_VALUES))); + } + + @Test(expected = NullPointerException.class) + public void testSingleValueNullBuffer() + { + system().singleValue((ByteBuffer) null); + } + + //======================================================================== + // Buffer position/limit contract (Requirement 5) + + @Test + public void testPositionAdvancedToLimitAfterCall() + { + for (BufferKind kind : BufferKind.values()) + { + ByteBuffer buffer = kind.wrap(binary(ONE_VALUE)); + int limitBefore = buffer.limit(); + int capacityBefore = buffer.capacity(); + + system().newReader(buffer); + + assertEquals("remaining, kind=" + kind, 0, buffer.remaining()); + assertEquals("position==limit, kind=" + kind, limitBefore, buffer.position()); + assertEquals("limit unchanged, kind=" + kind, limitBefore, buffer.limit()); + assertEquals("capacity unchanged, kind=" + kind, capacityBefore, buffer.capacity()); + } + } + + @Test + public void testMarkPreservedWhenStillValid() + { + // A mark set at position 0 remains valid after consuming to the limit, + // because reset() would move position back to 0 (<= limit). + ByteBuffer buffer = ByteBuffer.wrap(binary(ONE_VALUE)); + buffer.mark(); // mark at position 0 + + system().newReader(buffer); + assertEquals(0, buffer.remaining()); + + buffer.reset(); // must not throw InvalidMarkException + assertEquals("mark should restore position to 0", 0, buffer.position()); + } + + @Test + public void testSnapshotIsolationMutateAfterCall() + { + // The reader/value must operate over a private copy: mutating the + // (writable, array-backed) buffer's bytes after the call must not + // affect the already-extracted value. + byte[] data = text(ONE_VALUE); + ByteBuffer buffer = ByteBuffer.wrap(data); + + IonValue v = system().singleValue(buffer); + assertEquals("hello", ((IonString) v).stringValue()); + + // Corrupt the original backing array after the fact. + java.util.Arrays.fill(data, (byte) 'X'); + assertEquals("value must be unaffected by later buffer mutation", + "hello", ((IonString) v).stringValue()); + } + + @Test + public void testSubRangeReadsOnlyReadableRegion() + { + // The sliced buffer is surrounded by 0x7F sentinel bytes; if extraction + // read outside [position, limit) the parse would fail or produce wrong + // values. A correct read yields exactly the three ints. + ByteBuffer buffer = BufferKind.SLICED_SUBRANGE.wrap(binary(THREE_VALUES)); + Iterator it = system().iterate(buffer); + assertThreeInts(it, "SLICED_SUBRANGE"); + assertEquals(0, buffer.remaining()); + } + + //======================================================================== + // helpers + + private void assertThreeInts(Iterator it, String label) + { + for (int expected = 1; expected <= 3; expected++) + { + assertTrue(label + ": expected value " + expected, it.hasNext()); + IonValue v = it.next(); + assertEquals(label, expected, ((IonInt) v).intValue()); + } + assertFalse(label + ": no more values", it.hasNext()); + } +} diff --git a/src/test/java/com/amazon/ion/ReaderMaker.java b/src/test/java/com/amazon/ion/ReaderMaker.java index 74e6ac0572..34e1706e89 100644 --- a/src/test/java/com/amazon/ion/ReaderMaker.java +++ b/src/test/java/com/amazon/ion/ReaderMaker.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -134,6 +135,122 @@ public IonReader newReader(IonSystem system, byte[] ionData) }, + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with an + * array-backed {@link ByteBuffer} over Ion binary. + */ + FROM_BYTE_BUFFER_BINARY(Feature.BINARY) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureBinary(system, ionData); + return system.newReader(ByteBuffer.wrap(ionData)); + } + + @Override + public IonReader newReaderVerbatim(IonSystem system, String ionText) { + return system.newReader(ByteBuffer.wrap(convertToBinaryVerbatim(system, ionText))); + } + }, + + + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with an + * array-backed {@link ByteBuffer} over Ion text. + */ + FROM_BYTE_BUFFER_TEXT(Feature.TEXT) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureText(system, ionData); + return system.newReader(ByteBuffer.wrap(ionData)); + } + }, + + + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with a direct + * (off-heap) {@link ByteBuffer} over Ion binary, exercising the non-array + * code path. + */ + FROM_BYTE_BUFFER_DIRECT_BINARY(Feature.BINARY) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureBinary(system, ionData); + return system.newReader(directBuffer(ionData)); + } + + @Override + public IonReader newReaderVerbatim(IonSystem system, String ionText) { + return system.newReader(directBuffer(convertToBinaryVerbatim(system, ionText))); + } + }, + + + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with a read-only + * {@link ByteBuffer} over Ion binary, exercising the read-only code path. + */ + FROM_BYTE_BUFFER_READ_ONLY_BINARY(Feature.BINARY) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureBinary(system, ionData); + return system.newReader(ByteBuffer.wrap(ionData).asReadOnlyBuffer()); + } + + @Override + public IonReader newReaderVerbatim(IonSystem system, String ionText) { + return system.newReader(ByteBuffer.wrap(convertToBinaryVerbatim(system, ionText)).asReadOnlyBuffer()); + } + }, + + + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with a sliced + * {@link ByteBuffer} whose readable region is a sub-range of a larger + * backing array (non-zero {@code position} and {@code arrayOffset}), + * over Ion binary. Note: the resulting reader is created over a zero-based + * copy of the sub-range, so its octet offsets are stable at 0 (like + * {@link #FROM_BYTE_BUFFER_BINARY}); this maker verifies that sub-range + * extraction reads exactly the intended bytes. + */ + FROM_BYTE_BUFFER_OFFSET_BINARY(Feature.BINARY) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureBinary(system, ionData); + return system.newReader(slicedBuffer(ionData, 37, 70)); + } + + @Override + public IonReader newReaderVerbatim(IonSystem system, String ionText) { + return system.newReader(slicedBuffer(convertToBinaryVerbatim(system, ionText), 37, 70)); + } + }, + + + /** + * Invokes {@link IonSystem#newReader(java.nio.ByteBuffer)} with an + * array-backed {@link ByteBuffer} over Ion text (via a sliced sub-range). + */ + FROM_BYTE_BUFFER_OFFSET_TEXT(Feature.TEXT) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureText(system, ionData); + return system.newReader(slicedBuffer(ionData, 37, 70)); + } + }, + + /** * Invokes {@link IonSystem#newReader(InputStream)} with Ion binary. */ @@ -310,6 +427,40 @@ private static byte[] convertToBinaryVerbatim(IonSystem system, String ionText) return out.toByteArray(); } + /** + * Copies the given bytes into a direct (off-heap) {@link ByteBuffer} whose + * readable region is exactly {@code data}. + */ + private static ByteBuffer directBuffer(byte[] data) { + ByteBuffer buffer = ByteBuffer.allocateDirect(data.length); + buffer.put(data); + buffer.flip(); + return buffer; + } + + /** + * Places the given bytes into a larger array-backed buffer at + * {@code offset}, then returns a {@link ByteBuffer#slice()} whose readable + * region is exactly {@code data}. The slice has a non-zero + * {@code arrayOffset}, exercising sub-range extraction. + * + * @param data the payload bytes. + * @param offset the offset within the padded backing array at which the + * payload is placed. + * @param extraPadding total extra capacity (split before/after the payload) + * of the backing array. + */ + private static ByteBuffer slicedBuffer(byte[] data, int offset, int extraPadding) { + byte[] padded = new byte[data.length + extraPadding]; + System.arraycopy(data, 0, padded, offset, data.length); + ByteBuffer backing = ByteBuffer.wrap(padded); + backing.position(offset); + backing.limit(offset + data.length); + // slice() yields a buffer whose position is 0, limit/capacity are the + // payload length, and arrayOffset() reflects the sub-range start. + return backing.slice(); + } + public IonReader newReader(IonSystem system, byte[] ionData) { IonDatagram dg = system.getLoader().load(ionData);