feat(sdk): DSPX-4584 stream the TDF manifest, remove the ~21.7M segment ceiling

sdk/src/main/java/io/opentdf/platform/sdk/SDK.java

@@ -196,8 +196,9 @@ public static boolean isTDF(SeekableByteChannel channel) {
      */
     public static Manifest readManifest(SeekableByteChannel tdfBytes) throws SDKException, IOException {
         TDFReader reader = new TDFReader(tdfBytes);
-        String manifestJson = reader.manifest();
-        return Manifest.readManifest(manifestJson);
+        try (var manifestJson = reader.manifest()) {
+            return Manifest.readManifest(manifestJson);
+        }
     }
 
     /**

sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java

@@ -1,8 +1,10 @@
 package io.opentdf.platform.sdk;
 
-import java.io.ByteArrayOutputStream;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
@@ -19,6 +21,8 @@
  */
 public class TDFReader {
 
+    private static final int MANIFEST_BUFFER_SIZE = 1 << 16;
+
     private final ZipReader.Entry manifestEntry;
     private final InputStream payload;
 
@@ -49,15 +53,18 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException {
         payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData();
     }
 
-    String manifest() {
-        var out = new ByteArrayOutputStream();
+    /**
+     * The manifest entry as a character stream; the caller must close it. Returned
+     * as a stream rather than a String because a manifest with tens of millions of
+     * segments exceeds the maximum size of a Java String.
+     */
+    Reader manifest() {
         try {
-            manifestEntry.getData().transferTo(out);
+            return new BufferedReader(
+                    new InputStreamReader(manifestEntry.getData(), StandardCharsets.UTF_8), MANIFEST_BUFFER_SIZE);
         } catch (IOException e) {
             throw new SDKException("error retrieving manifest from zip file", e);
         }
-
-        return out.toString(StandardCharsets.UTF_8);
     }
 
     int readPayloadBytes(byte[] buf) {
@@ -74,8 +81,10 @@ int readPayloadBytes(byte[] buf) {
     }
 
     PolicyObject readPolicyObject() {
-        String manifestJson = manifest();
-        Manifest manifest = Manifest.readManifest(manifestJson);
-        return Manifest.decodePolicyObject(manifest);
+        try (Reader manifestRaw = manifest()) {
+            return Manifest.decodePolicyObject(Manifest.readManifest(manifestRaw));
+        } catch (IOException e) {
+            throw new SDKException("error reading manifest from zip file", e);
+        }
     }
 }

sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java

@@ -41,8 +41,25 @@ public TDFWriter(OutputStream destination) {
         this.archiveWriter = new ZipWriter(destination, maxNonZip64Value);
     }
 
+    /**
+     * @deprecated use {@link #manifest()}. A manifest with tens of millions of segments
+     *             exceeds the maximum size of a Java {@link String}, so this cannot
+     *             express every manifest the SDK writes. It produces the same zip entry.
+     */
+    @SuppressWarnings("java:S1133") // Retained for source and binary compatibility.
+    @Deprecated(since = "0.20.0", forRemoval = false)
     public void appendManifest(String manifest) throws IOException {
-        this.archiveWriter.data(TDF_MANIFEST_FILE_NAME, manifest.getBytes(StandardCharsets.UTF_8));
+        try (OutputStream output = manifest()) {
+            output.write(manifest.getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+    /**
+     * Opens the manifest entry for writing. The returned stream must be closed before
+     * {@link #finish()} is called, otherwise the entry never makes it into the central directory.
+     */
+    public OutputStream manifest() throws IOException {
+        return this.archiveWriter.stream(TDF_MANIFEST_FILE_NAME);
     }
 
     public OutputStream payload() throws IOException {
@@ -53,4 +70,4 @@ public OutputStream payload() throws IOException {
     public long finish() throws IOException {
         return this.archiveWriter.finish();
     }
-}
+}

sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java

@@ -1,13 +1,16 @@
 package io.opentdf.platform.sdk;
 
 import com.google.gson.Gson;
+import com.google.gson.JsonParseException;
 import com.google.gson.JsonParser;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.IOException;
+import java.io.EOFException;
+import java.io.Reader;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
@@ -271,6 +274,61 @@ void rejectsKeyAccessWithoutUrl() {
                 .hasMessage("Invalid KeyAccess with null url in manifest");
     }
 
+    @Test
+    void wrapsManifestReaderIoFailures() {
+        Reader failingReader = new Reader() {
+            @Override
+            public int read(char[] buffer, int offset, int length) throws IOException {
+                throw new IOException("manifest read failed");
+            }
+
+            @Override
+            public void close() {
+                // No resources to release.
+            }
+        };
+
+        assertThatThrownBy(() -> Manifest.readManifest(failingReader))
+                .isInstanceOf(SDKException.class)
+                .hasMessage("error reading manifest from zip file")
+                .hasRootCauseMessage("manifest read failed");
+    }
+
+    @Test
+    void preservesMalformedJsonErrors() {
+        assertThatThrownBy(() -> Manifest.readManifest("{\"payload\":"))
+                .isInstanceOf(JsonParseException.class)
+                .isNotInstanceOf(SDKException.class);
+    }
+
+    @Test
+    void wrapsEofThrownByTheSourceReader() {
+        Reader failingReader = new Reader() {
+            private final String partialManifest = "{\"payload\":";
+            private int position;
+
+            @Override
+            public int read(char[] buffer, int offset, int length) throws IOException {
+                if (position == partialManifest.length()) {
+                    throw new EOFException("archive ended early");
+                }
+                int copied = Math.min(length, partialManifest.length() - position);
+                partialManifest.getChars(position, position + copied, buffer, offset);
+                position += copied;
+                return copied;
+            }
+
+            @Override
+            public void close() {
+                // No resources to release.
+            }
+        };
+
+        assertThatThrownBy(() -> Manifest.readManifest(failingReader))
+                .isInstanceOf(SDKException.class)
+                .hasRootCauseMessage("archive ended early");
+    }
+
     @Test
     void testReadingManifestWithObjectStatementValue() throws IOException {
         final Manifest manifest;