From 4ec4243876fab7b6dc60b146c4cf234721018b04 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Fri, 16 Oct 2026 19:51:43 -0400 Subject: [PATCH 1/2] feat(sdk): load TDFs from HTTP(S) URLs via range requests (DSPX-5113) Add SDK.loadTDF(URI, TDFReaderConfig), backed by a read-only SeekableByteChannel where every read is one HTTP range request, so only the zip directory, the manifest and the payload segments actually read are downloaded. Mirrors the Go HTTPRangeReader (opentdf/platform#4221): - the size comes from a bytes=0-0 probe; 206 with a matching Content-Range is required, and a 200 or a mismatched range raises SDK.RangeNotSupportedException - 3xx is reported as "redirects are not followed, use the final URL"; other statuses report their own status - a total size that changes between requests is rejected - requests go through an unauthenticated client that keeps the SDK's TLS settings, so platform credentials are never sent to the TDF host The manifest is now read in 1 MiB chunks so a large manifest isn't fetched in small range requests. Signed-off-by: sujan kota --- .../platform/sdk/HttpRangeChannel.java | 152 ++++++++++++++++++ .../java/io/opentdf/platform/sdk/SDK.java | 37 +++++ .../io/opentdf/platform/sdk/SDKBuilder.java | 9 +- .../io/opentdf/platform/sdk/TDFReader.java | 10 +- .../platform/sdk/HttpRangeChannelTest.java | 130 +++++++++++++++ 5 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java create mode 100644 sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java b/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java new file mode 100644 index 00000000..1bcaefc2 --- /dev/null +++ b/sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java @@ -0,0 +1,152 @@ +package io.opentdf.platform.sdk; + +import okhttp3.OkHttpClient; +import okhttp3.Request; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.util.regex.Pattern; + +/** + * A read-only {@link SeekableByteChannel} over a remote resource, where every read is one + * HTTP range request. The size is learned by probing {@code bytes=0-0}, and every later + * response must report the same total size, so a resource that changes underneath the + * reader is rejected rather than spliced together. + *

+ * Redirects are not followed: the client handed in must have redirects disabled, and a 3xx + * is reported as an error so the caller can use the final URL. + */ +// ponytail: no read cache, so ZipReader's small header reads cost a few dozen requests per +// load; add a read-ahead window if that shows up over a real network +class HttpRangeChannel implements SeekableByteChannel { + private static final Pattern CONTENT_RANGE = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)"); + + private final OkHttpClient client; + private final URI url; + private final long size; + private long position; + private boolean open = true; + + HttpRangeChannel(OkHttpClient client, URI url) throws IOException { + this.client = client; + this.url = url; + this.size = fetch(ByteBuffer.allocate(1), 0); + } + + @Override + public int read(ByteBuffer dst) throws IOException { + ensureOpen(); + if (position >= size) { + return -1; + } + int len = (int) Math.min(dst.remaining(), size - position); + if (len == 0) { + return 0; + } + var window = dst.slice(); + window.limit(len); + long total = fetch(window, position); + if (total != size) { + throw new IOException("remote resource size changed from " + size + " to " + total + " bytes"); + } + dst.position(dst.position() + len); + position += len; + return len; + } + + /** + * Fills {@code dst} with the bytes starting at {@code offset} and returns the total size + * of the resource as reported by the server. + */ + private long fetch(ByteBuffer dst, long offset) throws IOException { + long end = offset + dst.remaining() - 1; + var request = new Request.Builder() + .url(url.toString()) + .header("Range", "bytes=" + offset + "-" + end) + .build(); + try (var response = client.newCall(request).execute()) { + int status = response.code(); + if (status == 200) { + throw new SDK.RangeNotSupportedException( + "status 200 for bytes " + offset + "-" + end); + } + if (status >= 300 && status < 400) { + throw new IOException("range request redirected (" + status + + "); redirects are not followed, use the final URL"); + } + if (status != 206) { + throw new IOException("range request for bytes " + offset + "-" + end + " failed: " + status); + } + String contentRange = response.header("Content-Range"); + var m = contentRange == null ? null : CONTENT_RANGE.matcher(contentRange); + if (m == null || !m.matches()) { + throw new SDK.RangeNotSupportedException("invalid Content-Range " + contentRange); + } + long gotStart = Long.parseLong(m.group(1)); + long gotEnd = Long.parseLong(m.group(2)); + if (gotStart != offset || gotEnd != end) { + throw new SDK.RangeNotSupportedException("requested bytes " + offset + "-" + end + + ", got " + gotStart + "-" + gotEnd); + } + var source = response.body().source(); + while (dst.hasRemaining()) { + if (source.read(dst) < 0) { + throw new IOException("range response for bytes " + offset + "-" + end + " ended early"); + } + } + return Long.parseLong(m.group(3)); + } + } + + @Override + public long position() throws IOException { + ensureOpen(); + return position; + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + ensureOpen(); + if (newPosition < 0) { + throw new IllegalArgumentException("negative position " + newPosition); + } + position = newPosition; + return this; + } + + @Override + public long size() throws IOException { + ensureOpen(); + return size; + } + + @Override + public int write(ByteBuffer src) { + throw new NonWritableChannelException(); + } + + @Override + public SeekableByteChannel truncate(long size) { + throw new NonWritableChannelException(); + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() { + open = false; + } + + private void ensureOpen() throws ClosedChannelException { + if (!open) { + throw new ClosedChannelException(); + } + } +} diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index f69b6896..041d74de 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; import java.nio.channels.SeekableByteChannel; import java.util.ArrayList; import java.util.Collections; @@ -35,6 +36,8 @@ import java.util.Optional; import java.util.stream.Collectors; +import okhttp3.OkHttpClient; + /** * The SDK class represents a software development kit for interacting with the * opentdf platform. It @@ -57,6 +60,8 @@ public class SDK implements AutoCloseable { private final String platformUrl; private final ProtocolClient platformServicesClient; private final SrtSigner srtSigner; + // Fetches TDFs by URL. Unauthenticated: platform credentials never go to the TDF host. + private final OkHttpClient rangeClient; /** * Closes the SDK, including its associated services. @@ -66,6 +71,8 @@ public class SDK implements AutoCloseable { @Override public void close() throws Exception { services.close(); + rangeClient.dispatcher().executorService().shutdown(); + rangeClient.connectionPool().evictAll(); } /** @@ -114,6 +121,12 @@ public Optional getAuthInterceptor() { } SDK(Services services, TrustManager trustManager, Interceptor authInterceptor, ProtocolClient platformServicesClient, String platformUrl, SrtSigner srtSigner) { + this(services, trustManager, authInterceptor, platformServicesClient, platformUrl, srtSigner, + new OkHttpClient.Builder().followRedirects(false).followSslRedirects(false).build()); + } + + SDK(Services services, TrustManager trustManager, Interceptor authInterceptor, ProtocolClient platformServicesClient, String platformUrl, SrtSigner srtSigner, OkHttpClient rangeClient) { + this.rangeClient = rangeClient; this.platformUrl = platformUrl; this.services = services; this.trustManager = trustManager; @@ -140,6 +153,20 @@ public TDF.Reader loadTDF(SeekableByteChannel channel, Config.TDFReaderConfig co return tdf.loadTDF(channel, config, platformUrl); } + /** + * Loads a TDF served over HTTP(S), reading it with range requests so only the zip + * directory, the manifest, and the payload segments actually read are downloaded. + * The server must answer range requests with 206; redirects are not followed. + * + * @throws RangeNotSupportedException if the server ignores range requests + */ + public TDF.Reader loadTDF(URI url, Config.TDFReaderConfig config) throws SDKException, IOException { + if (!"http".equalsIgnoreCase(url.getScheme()) && !"https".equalsIgnoreCase(url.getScheme())) { + throw new IllegalArgumentException("unsupported URL scheme: " + url.getScheme()); + } + return loadTDF(new HttpRangeChannel(rangeClient, url), config); + } + public Manifest createTDF(InputStream payload, OutputStream outputStream, Config.TDFConfig config) throws SDKException, IOException { var tdf = new TDF(services); return tdf.createTDF(payload, outputStream, config).getManifest(); @@ -381,6 +408,16 @@ public boolean attributeValueExists(String valueFqn) { return resp.getFqnAttributeValuesMap().containsKey(valueFqn); } + /** + * The server holding a TDF did not honor an HTTP range request, so the TDF cannot be + * read in place; download it and load it from a file instead. + */ + public static class RangeNotSupportedException extends IOException { + public RangeNotSupportedException(String errorMessage) { + super("server did not honor the HTTP range request: " + errorMessage); + } + } + /** * Indicates that the TDF is malformed in some way */ diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java index dda13258..cac363d0 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDKBuilder.java @@ -483,7 +483,14 @@ private KASClient getKASClient(SrtSigner srtSigner, AuthInterceptor interceptor, public SDK build() { var services = buildServices(); - return new SDK(services.services, services.trustManager, services.interceptor, services.protocolClient, platformEndpoint, services.srtSigner); + return new SDK(services.services, services.trustManager, services.interceptor, services.protocolClient, platformEndpoint, services.srtSigner, + // getHttpClient() carries the TLS settings but no credentials. Drop the h2c prior + // knowledge plaintext mode forces, since TDF hosts are ordinary HTTP servers. + getHttpClient().newBuilder() + .protocols(List.of(Protocol.HTTP_2, Protocol.HTTP_1_1)) + .followRedirects(false) + .followSslRedirects(false) + .build()); } private ProtocolClient getUnauthenticatedProtocolClient(String endpoint, OkHttpClient httpClient) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 93548d67..50deed83 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -51,8 +51,14 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { String manifest() { var out = new ByteArrayOutputStream(); - try { - manifestEntry.getData().transferTo(out); + // Read in large chunks rather than transferTo's small ones: over HTTP every read + // is a range request. + var buf = new byte[1 << 20]; + try (var in = manifestEntry.getData()) { + int n; + while ((n = in.read(buf)) >= 0) { + out.write(buf, 0, n); + } } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java new file mode 100644 index 00000000..9c0ab181 --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java @@ -0,0 +1,130 @@ +package io.opentdf.platform.sdk; + +import okhttp3.OkHttpClient; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okio.Buffer; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HttpRangeChannelTest { + static final OkHttpClient CLIENT = new OkHttpClient.Builder() + .followRedirects(false).followSslRedirects(false).build(); + + private static final Pattern RANGE = Pattern.compile("bytes=(\\d+)-(\\d+)"); + + /** Serves {@code data} the way a static file server does, adding the bytes sent to {@code served}. */ + static Dispatcher rangeDispatcher(Supplier data, AtomicLong served) { + return new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + byte[] bytes = data.get(); + var m = RANGE.matcher(String.valueOf(request.getHeader("Range"))); + if (!m.matches()) { + return new MockResponse().setResponseCode(200).setBody(new Buffer().write(bytes)); + } + int start = Integer.parseInt(m.group(1)); + int end = Math.min(Integer.parseInt(m.group(2)), bytes.length - 1); + served.addAndGet(end - start + 1); + return new MockResponse().setResponseCode(206) + .setHeader("Content-Range", "bytes " + start + "-" + end + "/" + bytes.length) + .setBody(new Buffer().write(bytes, start, end - start + 1)); + } + }; + } + + private static URI url(MockWebServer server) { + return server.url("/x.tdf").uri(); + } + + @Test + void readsAndSeeks() throws Exception { + byte[] data = "0123456789".getBytes(StandardCharsets.UTF_8); + try (var server = new MockWebServer()) { + server.setDispatcher(rangeDispatcher(() -> data, new AtomicLong())); + try (var ch = new HttpRangeChannel(CLIENT, url(server))) { + assertThat(ch.size()).isEqualTo(10); + + var buf = ByteBuffer.allocate(4); + assertThat(ch.position(3).read(buf)).isEqualTo(4); + assertThat(new String(buf.array(), StandardCharsets.UTF_8)).isEqualTo("3456"); + assertThat(ch.position()).isEqualTo(7); + + // clamped at the end of the resource, then EOF + buf.clear(); + assertThat(ch.read(buf)).isEqualTo(3); + assertThat(ch.read(buf.clear())).isEqualTo(-1); + } + assertThat(server.takeRequest().getHeader("Authorization")).isNull(); + } + } + + @Test + void rangeNotSupported() throws Exception { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(200).setBody("not a partial response")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(SDK.RangeNotSupportedException.class); + } + } + + @Test + void mismatchedContentRangeIsRangeNotSupported() throws Exception { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(206) + .setHeader("Content-Range", "bytes 0-9/10").setBody("0123456789")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(SDK.RangeNotSupportedException.class); + } + } + + @Test + void errorStatusesReportTheirStatus() throws Exception { + for (int status : new int[] { 302, 403, 404 }) { + try (var server = new MockWebServer()) { + server.enqueue(new MockResponse().setResponseCode(status).setHeader("Location", "/elsewhere")); + assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server))) + .isInstanceOf(IOException.class) + .isNotInstanceOf(SDK.RangeNotSupportedException.class) + .hasMessageContaining(String.valueOf(status)) + .satisfies(e -> { + if (status == 302) { + assertThat(e).hasMessageContaining("redirects are not followed"); + } + }); + } + } + } + + @Test + void sizeChanged() throws Exception { + var data = new byte[][] { "0123456789".getBytes(StandardCharsets.UTF_8) }; + try (var server = new MockWebServer()) { + server.setDispatcher(rangeDispatcher(() -> data[0], new AtomicLong())); + try (var ch = new HttpRangeChannel(CLIENT, url(server))) { + data[0] = "0123456789abc".getBytes(StandardCharsets.UTF_8); + assertThatThrownBy(() -> ch.read(ByteBuffer.allocate(4))) + .hasMessageContaining("size changed from 10 to 13"); + } + } + } + + @Test + void rejectsNonHttpSchemes() { + var sdk = new SDK(new FakeServicesBuilder().build(), null, null, null, null, null); + assertThatThrownBy(() -> sdk.loadTDF(URI.create("file:///etc/passwd"), Config.newTDFReaderConfig())) + .isInstanceOf(IllegalArgumentException.class); + } +} From 8c27028a3f4390032c39eb3e7a1ca3d789096906 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Fri, 16 Oct 2026 19:57:25 -0400 Subject: [PATCH 2/2] feat(sdk,cmdline): random-access plaintext reads and decrypt --offset/--length (DSPX-5115) Add TDF.Reader.readPayload(out, offset, length), which reads, verifies and decrypts only the segments covering the requested plaintext range. Segments before the range are skipped without reading them, so over an HTTP range source only the covering segments are downloaded. A segment ending exactly at the offset is skipped (<=, per opentdf/platform#4233). Range semantics match otdfctl (opentdf/platform#4229): a negative length reads to the end, a range past the end is clamped, an offset equal to the payload size writes nothing, and an offset past the end fails before anything is written. Segment plaintext sizes are derived from the ciphertext length instead of the manifest's segmentSize. Each call opens a fresh payload stream, so a reader can serve several ranges. ZipReader entry streams now skip by moving the cursor instead of reading and discarding. cmdline decrypt: - -f accepts an http(s):// URL, loaded with range requests - --offset (default 0) and --length (default -1) - -o/--output writes to a sibling temp file and moves it into place only on success, so a failed decrypt leaves no file - supports reports partial_decrypt and http_range_source Signed-off-by: sujan kota --- .../java/io/opentdf/platform/Command.java | 126 +++++++++++------- .../java/io/opentdf/platform/CommandTest.java | 12 +- .../java/io/opentdf/platform/sdk/TDF.java | 117 ++++++++++++---- .../io/opentdf/platform/sdk/TDFReader.java | 13 +- .../io/opentdf/platform/sdk/ZipReader.java | 9 ++ .../java/io/opentdf/platform/sdk/TDFTest.java | 91 +++++++++++++ 6 files changed, 293 insertions(+), 75 deletions(-) diff --git a/cmdline/src/main/java/io/opentdf/platform/Command.java b/cmdline/src/main/java/io/opentdf/platform/Command.java index b073b955..0f6b57f8 100644 --- a/cmdline/src/main/java/io/opentdf/platform/Command.java +++ b/cmdline/src/main/java/io/opentdf/platform/Command.java @@ -15,12 +15,15 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.nio.channels.FileChannel; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; @@ -77,7 +80,8 @@ class Command { static class Supports implements Callable { // Static, always-on capabilities of this build. There is no "known but // unsupported" feature here: anything not in this list is simply unrecognized. - private static final List FEATURES = List.of("dpop", "dpop_nonce_challenge"); + private static final List FEATURES = List.of("dpop", "dpop_nonce_challenge", + "partial_decrypt", "http_range_source"); @CommandLine.Parameters(index = "0", arity = "0..1", description = "Feature to check (e.g., dpop). Omit to list all supported features.") private String feature; @@ -354,7 +358,10 @@ private SDK buildSDK() { @CommandLine.Command(name = "decrypt") void decrypt( - @Option(names = { "-f", "--file" }, required = true) Path tdfPath, + @Option(names = { "-f", "--file" }, required = true, description = "TDF file path, or an http(s):// URL read with range requests") String tdfFile, + @Option(names = { "-o", "--output" }, defaultValue = Option.NULL_VALUE, description = "Write plaintext here, only on success (default: stdout)") Optional output, + @Option(names = { "--offset" }, defaultValue = "0", description = "Plaintext byte offset to start decrypting from") long offset, + @Option(names = { "--length" }, defaultValue = "-1", description = "Plaintext bytes to decrypt from --offset; negative decrypts to the end") long length, @Option(names = { "--rewrap-key-type" }, defaultValue = Option.NULL_VALUE, description = "Preferred rewrap algorithm, one of ${COMPLETION-CANDIDATES}") Optional rewrapKeyType, @Option(names = { @@ -367,60 +374,89 @@ void decrypt( throws Exception { try (var sdk = buildSDK()) { var opts = new ArrayList>(); - try (var in = FileChannel.open(tdfPath, StandardOpenOption.READ)) { - try (var stdout = new BufferedOutputStream(System.out)) { - if (assertionVerification.isPresent()) { - var assertionVerificationInput = assertionVerification.get(); - Gson gson = buildGson(); - - Config.AssertionVerificationKeys assertionVerificationKeys; - try { - assertionVerificationKeys = gson.fromJson(assertionVerificationInput, - Config.AssertionVerificationKeys.class); - } catch (JsonSyntaxException e) { - // try it as a file path - try { - String fileJson = Files.readString(Paths.get(assertionVerificationInput)); - assertionVerificationKeys = gson.fromJson(fileJson, - Config.AssertionVerificationKeys.class); - } catch (JsonSyntaxException e2) { - throw new RuntimeException("Failed to parse assertion verification keys from file", e2); - } catch (Exception e3) { - throw new RuntimeException( - "Could not parse assertion verification keys as json string or path to file", - e3); - } - } - - for (Map.Entry entry : assertionVerificationKeys.keys - .entrySet()) { - try { - Object correctedKey = correctKeyType(entry.getValue().alg, entry.getValue().key, true); - entry.setValue(new AssertionConfig.AssertionKey(entry.getValue().alg, correctedKey)); - } catch (Exception e) { - throw new RuntimeException("Error with assertion verification key: " + e.getMessage(), - e); - } - } - opts.add(Config.withAssertionVerificationKeys(assertionVerificationKeys)); + if (assertionVerification.isPresent()) { + var assertionVerificationInput = assertionVerification.get(); + Gson gson = buildGson(); + + Config.AssertionVerificationKeys assertionVerificationKeys; + try { + assertionVerificationKeys = gson.fromJson(assertionVerificationInput, + Config.AssertionVerificationKeys.class); + } catch (JsonSyntaxException e) { + // try it as a file path + try { + String fileJson = Files.readString(Paths.get(assertionVerificationInput)); + assertionVerificationKeys = gson.fromJson(fileJson, + Config.AssertionVerificationKeys.class); + } catch (JsonSyntaxException e2) { + throw new RuntimeException("Failed to parse assertion verification keys from file", e2); + } catch (Exception e3) { + throw new RuntimeException( + "Could not parse assertion verification keys as json string or path to file", + e3); } + } - if (disableAssertionVerification) { - opts.add(Config.withDisableAssertionVerification(true)); + for (Map.Entry entry : assertionVerificationKeys.keys + .entrySet()) { + try { + Object correctedKey = correctKeyType(entry.getValue().alg, entry.getValue().key, true); + entry.setValue(new AssertionConfig.AssertionKey(entry.getValue().alg, correctedKey)); + } catch (Exception e) { + throw new RuntimeException("Error with assertion verification key: " + e.getMessage(), + e); } - rewrapKeyType.map(Config::WithSessionKeyType).ifPresent(opts::add); + } + opts.add(Config.withAssertionVerificationKeys(assertionVerificationKeys)); + } + + if (disableAssertionVerification) { + opts.add(Config.withDisableAssertionVerification(true)); + } + rewrapKeyType.map(Config::WithSessionKeyType).ifPresent(opts::add); - ignoreAllowlist.ifPresent(aBoolean -> opts.add(Config.WithIgnoreKasAllowlist(aBoolean))); - kasAllowlistStr.ifPresent(s -> opts.add(Config.WithKasAllowlist(s.split(",")))); + ignoreAllowlist.ifPresent(aBoolean -> opts.add(Config.WithIgnoreKasAllowlist(aBoolean))); + kasAllowlistStr.ifPresent(s -> opts.add(Config.WithKasAllowlist(s.split(",")))); - var readerConfig = Config.newTDFReaderConfig(opts.toArray(new Consumer[0])); + var readerConfig = Config.newTDFReaderConfig(opts.toArray(new Consumer[0])); + + if (tdfFile.startsWith("http://") || tdfFile.startsWith("https://")) { + var reader = sdk.loadTDF(URI.create(tdfFile), readerConfig); + writePlaintext(out -> reader.readPayload(out, offset, length), output); + } else { + try (var in = FileChannel.open(Path.of(tdfFile), StandardOpenOption.READ)) { var reader = sdk.loadTDF(in, readerConfig); - reader.readPayload(stdout); + writePlaintext(out -> reader.readPayload(out, offset, length), output); } } } } + // With an output file, write to a sibling temp file and move it into place only on + // success, so a failed decrypt (e.g. an offset past the end) leaves no file behind. + private interface PlaintextWriter { + void writeTo(OutputStream out) throws Exception; + } + + private static void writePlaintext(PlaintextWriter plaintext, Optional output) throws Exception { + if (output.isEmpty()) { + try (var stdout = new BufferedOutputStream(System.out)) { + plaintext.writeTo(stdout); + } + return; + } + Path target = output.get().toAbsolutePath(); + Path tmp = Files.createTempFile(target.getParent(), target.getFileName() + ".", ".tmp"); + try { + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(tmp))) { + plaintext.writeTo(out); + } + Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } finally { + Files.deleteIfExists(tmp); + } + } + @CommandLine.Command(name = "metadata") void readMetadata( @Option(names = { "-f", "--file" }, required = true) Path tdfPath, diff --git a/cmdline/src/test/java/io/opentdf/platform/CommandTest.java b/cmdline/src/test/java/io/opentdf/platform/CommandTest.java index 8995271d..aa95a5b0 100644 --- a/cmdline/src/test/java/io/opentdf/platform/CommandTest.java +++ b/cmdline/src/test/java/io/opentdf/platform/CommandTest.java @@ -44,6 +44,12 @@ void supports_dpop_nonce_challenge_exits_0() { assertThat(code).isEqualTo(0); } + @Test + void supports_partial_decrypt_and_http_range_source_exit_0() { + assertThat(new CommandLine(new Command()).execute("supports", "partial_decrypt")).isEqualTo(0); + assertThat(new CommandLine(new Command()).execute("supports", "http_range_source")).isEqualTo(0); + } + @Test void supports_unknown_feature_exits_1() { int code = new CommandLine(new Command()).execute("supports", "unknown_feature"); @@ -95,7 +101,8 @@ void supports_noArgs_listsFeatures_exits0() { String out = captureStdout(() -> code[0] = new CommandLine(new Command()).execute("supports")); assertThat(code[0]).isEqualTo(0); - assertThat(out.lines()).containsExactlyInAnyOrder("dpop", "dpop_nonce_challenge"); + assertThat(out.lines()).containsExactlyInAnyOrder("dpop", "dpop_nonce_challenge", + "partial_decrypt", "http_range_source"); } @Test @@ -104,7 +111,8 @@ void supports_noArgs_json_listsFeatures() { String out = captureStdout(() -> code[0] = new CommandLine(new Command()).execute("supports", "--json")); assertThat(code[0]).isEqualTo(0); - assertThat(out.trim()).isEqualTo("{\"dpop\":true,\"dpop_nonce_challenge\":true}"); + assertThat(out.trim()).isEqualTo("{\"dpop\":true,\"dpop_nonce_challenge\":true," + + "\"partial_decrypt\":true,\"http_range_source\":true}"); } @Test diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index fddead94..19c58b4e 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -388,6 +388,41 @@ public Manifest getManifest() { } public void readPayload(OutputStream outputStream) throws SDK.TamperException, IOException { + readPayload(outputStream, 0, -1); + } + + /** + * Writes the plaintext bytes {@code [offset, offset + length)} to {@code outputStream}. + * Only the segments covering the range are read, verified and decrypted, so over + * {@link SDK#loadTDF(java.net.URI, Config.TDFReaderConfig)} only those are downloaded. + * + * @param offset plaintext offset to start at; equal to the payload size writes nothing + * @param length number of bytes to write; negative means to the end of the payload, and + * a range running past the end is clamped to it + * @throws IllegalArgumentException if {@code offset} is negative or beyond the end of the + * payload, before anything is written + */ + public void readPayload(OutputStream outputStream, long offset, long length) + throws SDK.TamperException, IOException { + if (offset < 0) { + throw new IllegalArgumentException("offset must not be negative: " + offset); + } + + validateSegmentSizes(); + + List segments = manifest.encryptionInformation.integrityInformation.segments; + long total = 0; + for (Manifest.Segment segment : segments) { + total += plaintextSize(segment); + } + if (offset > total) { + throw new IllegalArgumentException( + "offset " + offset + " is beyond the end of the payload (" + total + " bytes)"); + } + long end = length < 0 || length > total - offset ? total : offset + length; + if (offset == end) { + return; + } MessageDigest digest = null; try { @@ -396,43 +431,71 @@ public void readPayload(OutputStream outputStream) throws SDK.TamperException, I throw new IllegalStateException("error getting instance of SHA-256", e); } - validateSegmentSizes(); - - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; - int bytesRead = tdfReader.readPayloadBytes(readBuf); + try (var payload = tdfReader.payload()) { + long plainStart = 0; + for (Manifest.Segment segment : segments) { + if (plainStart >= end) { + break; + } + long plainEnd = plainStart + plaintextSize(segment); + // <=, not <: a segment ending exactly at offset holds none of the range + if (plainEnd <= offset) { + if (payload.skip(segment.encryptedSegmentSize) != segment.encryptedSegmentSize) { + throw new IllegalStateException("unable to skip segment of " + + segment.encryptedSegmentSize + " bytes"); + } + plainStart = plainEnd; + continue; + } - if (readBuf.length != bytesRead) { - throw new IllegalStateException("unable to read bytes for segment (wanted " - + segment.encryptedSegmentSize + " but got " + bytesRead + ")"); + byte[] plain = readSegment(payload, segment, digest); + int from = (int) Math.max(0, offset - plainStart); + int to = (int) Math.min(plain.length, end - plainStart); + outputStream.write(plain, from, to - from); + plainStart = plainEnd; } + } + } - var isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); + // Derived from the ciphertext length rather than trusting the manifest's segmentSize, + // so a range lands on the bytes decryption actually produces. + private long plaintextSize(Manifest.Segment segment) { + return segment.encryptedSegmentSize + - (manifest.payload.isEncrypted ? kGcmIvSize + AesGcm.GCM_TAG_LENGTH : 0); + } - if (manifest.payload.isEncrypted) { - var sigAlg = segmentIntegrityAlgorithmFromManifest( - manifest.encryptionInformation.integrityInformation.segmentHashAlg); + private byte[] readSegment(InputStream payload, Manifest.Segment segment, MessageDigest digest) { + byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; + int bytesRead = TDFReader.readPayloadBytes(payload, readBuf); - var payloadSig = segmentIntegrity(readBuf, payloadKey, sigAlg); - if (isLegacyTdf) { - payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8); - } + if (readBuf.length != bytesRead) { + throw new IllegalStateException("unable to read bytes for segment (wanted " + + segment.encryptedSegmentSize + " but got " + bytesRead + ")"); + } - if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSig)) != 0) { - throw new SDK.SegmentSignatureMismatch("segment signature miss match"); - } + var isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); - byte[] writeBuf = aesGcm.decrypt(new AesGcm.Encrypted(readBuf)); - outputStream.write(writeBuf); + if (manifest.payload.isEncrypted) { + var sigAlg = segmentIntegrityAlgorithmFromManifest( + manifest.encryptionInformation.integrityInformation.segmentHashAlg); - } else { - String segmentSig = Hex.encodeHexString(digest.digest(readBuf)); - if (segment.hash.compareTo(segmentSig) != 0) { - throw new SDK.SegmentSignatureMismatch("segment signature miss match"); - } + var payloadSig = segmentIntegrity(readBuf, payloadKey, sigAlg); + if (isLegacyTdf) { + payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8); + } - outputStream.write(readBuf); + if (segment.hash.compareTo(Base64.getEncoder().encodeToString(payloadSig)) != 0) { + throw new SDK.SegmentSignatureMismatch("segment signature miss match"); } + + return aesGcm.decrypt(new AesGcm.Encrypted(readBuf)); + } else { + String segmentSig = Hex.encodeHexString(digest.digest(readBuf)); + if (segment.hash.compareTo(segmentSig) != 0) { + throw new SDK.SegmentSignatureMismatch("segment signature miss match"); + } + + return readBuf; } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 50deed83..80ad7e34 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -20,6 +20,7 @@ public class TDFReader { private final ZipReader.Entry manifestEntry; + private final ZipReader.Entry payloadEntry; private final InputStream payload; public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { @@ -46,7 +47,8 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { } manifestEntry = manifest; - payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); + payloadEntry = entries.get(TDF_PAYLOAD_FILE_NAME); + payload = payloadEntry.getData(); } String manifest() { @@ -66,7 +68,16 @@ String manifest() { return out.toString(StandardCharsets.UTF_8); } + /** A fresh stream over the payload, positioned at its start. */ + InputStream payload() throws IOException { + return payloadEntry.getData(); + } + int readPayloadBytes(byte[] buf) { + return readPayloadBytes(payload, buf); + } + + static int readPayloadBytes(InputStream payload, byte[] buf) { int totalRead = 0; int nread; try { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java index 576ba5de..d5e3a043 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/ZipReader.java @@ -254,6 +254,15 @@ private boolean doneReading() { return offset >= fileSize; } + // Moves the cursor without reading: the default skip reads and discards, which + // over HTTP would download everything skipped. + @Override + public long skip(long n) { + long skipped = Math.max(0, Math.min(n, fileSize - offset)); + offset += skipped; + return skipped; + } + private void setChannelPosition() throws IOException { var nextPosition = startPosition + offset; if (zipChannel.position() != nextPosition) { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java index aa5b38e6..3d85c6d2 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java @@ -13,10 +13,12 @@ import io.opentdf.platform.policy.kasregistry.ListKeyAccessServersRequest; import io.opentdf.platform.policy.kasregistry.ListKeyAccessServersResponse; import io.opentdf.platform.sdk.TDF.Reader; +import okhttp3.mockwebserver.MockWebServer; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nonnull; @@ -37,6 +39,7 @@ import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -733,6 +736,94 @@ public void testCreatingTDFWithMultipleSegments() throws Exception { assertThat(reader.getMetadata()).isEqualTo(metadata); } + private static final int SEG = Config.MIN_SEGMENT_SIZE; + + /** A TDF of {@code 2.5 * SEG} random bytes, so it has two full segments and a partial one. */ + private static byte[][] partialReadFixture() throws Exception { + var data = new byte[SEG * 5 / 2]; + new Random().nextBytes(data); + var tdfOutputStream = new ByteArrayOutputStream(); + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + tdf.createTDF(new ByteArrayInputStream(data), tdfOutputStream, Config.newTDFConfig( + Config.withAutoconfigure(false), + Config.withKasInformation(getSingleRSAKASInfo()), + Config.withSegmentSize(SEG))); + return new byte[][] { data, tdfOutputStream.toByteArray() }; + } + + private static Reader loadFixture(byte[] tdfBytes) throws Exception { + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + return tdf.loadTDF(new SeekableInMemoryByteChannel(tdfBytes), platformUrl); + } + + @ParameterizedTest + @CsvSource({ + "0, -1", // whole payload + "16334, 100", // crosses the first segment boundary + "16384, 100", // starts exactly where a segment ends + "16384, 16384", // exactly the second segment + "100, 0", // empty + "32768, -1", // last segment to the end + "40950, 1000", // runs past the end: clamped + "40960, -1", // offset == size: empty + }) + void readPayloadRange(long offset, long length) throws Exception { + var fixture = partialReadFixture(); + byte[] data = fixture[0]; + var out = new ByteArrayOutputStream(); + loadFixture(fixture[1]).readPayload(out, offset, length); + + int end = length < 0 ? data.length : (int) Math.min(data.length, offset + length); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(data, (int) offset, end)); + } + + @Test + void readPayloadRangeCanBeRepeatedOnOneReader() throws Exception { + var fixture = partialReadFixture(); + var reader = loadFixture(fixture[1]); + for (int i = 0; i < 2; i++) { + var out = new ByteArrayOutputStream(); + reader.readPayload(out, SEG, 10); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(fixture[0], SEG, SEG + 10)); + } + } + + @Test + void readPayloadRangeRejectsBadOffsetsBeforeWriting() throws Exception { + var fixture = partialReadFixture(); + var reader = loadFixture(fixture[1]); + var out = new ByteArrayOutputStream(); + assertThatThrownBy(() -> reader.readPayload(out, fixture[0].length + 1L, -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("beyond the end"); + assertThatThrownBy(() -> reader.readPayload(out, -1, -1)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(out.size()).isZero(); + } + + @Test + void readPayloadRangeOverHttpFetchesOnlyCoveringSegments() throws Exception { + var fixture = partialReadFixture(); + var served = new AtomicLong(); + try (var server = new MockWebServer()) { + server.setDispatcher(HttpRangeChannelTest.rangeDispatcher(() -> fixture[1], served)); + var tdf = new TDF(new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + var reader = tdf.loadTDF(new HttpRangeChannel(HttpRangeChannelTest.CLIENT, server.url("/x.tdf").uri()), + Config.newTDFReaderConfig(), platformUrl); + + served.set(0); + var out = new ByteArrayOutputStream(); + reader.readPayload(out, 2L * SEG - 50, 100); + assertThat(out.toByteArray()).containsExactly(Arrays.copyOfRange(fixture[0], 2 * SEG - 50, 2 * SEG + 50)); + // the second (full) and third (half) segments, plus the payload entry's local header + long encryptedSegments = (SEG + 28) + (SEG / 2 + 28); + assertThat(served.get()).isBetween(encryptedSegments, encryptedSegments + 1024); + } + } + /** * A segment size small enough that the test payloads below span several segments. Matches * web-sdk's {@code DEFAULT_SEGMENT_SIZE}, though nothing here depends on the exact value.