From a5e504bfb71dbc515158d88e3202c33f2d11a9e7 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Wed, 9 Sep 2026 12:40:23 -0400 Subject: [PATCH 1/5] feat(sdk): stream the TDF manifest, remove the ~21.7M segment ceiling (DSPX-4584) The manifest was assembled entirely in memory on both paths, so a payload past ~21.7M segments (41 TiB at 2 MiB segments) hard-failed with OutOfMemoryError: Requested array size exceeds VM limit. - serialize the manifest straight into the zip entry and parse it from a Reader, so neither side materializes it as a Java String - add Manifest.Segments, a compact fixed-stride AbstractList backed by packed hash bytes (~24 B/segment instead of ~176), with an ArrayList fallback for manifests this SDK did not write - materialize the aggregate hash once and share it with the root signature and every assertion instead of copying it per assertion Manifest JSON bytes are unchanged. The 0.manifest.json zip entry is now a streamed data-descriptor entry, the same form 0.payload already used. EOF --- .../io/opentdf/platform/sdk/Manifest.java | 213 +++++++++++++++++- .../java/io/opentdf/platform/sdk/SDK.java | 5 +- .../java/io/opentdf/platform/sdk/TDF.java | 93 +++++--- .../io/opentdf/platform/sdk/TDFReader.java | 27 ++- .../io/opentdf/platform/sdk/TDFWriter.java | 9 +- .../io/opentdf/platform/sdk/SegmentsTest.java | 213 ++++++++++++++++++ .../opentdf/platform/sdk/TDFWriterTest.java | 58 ++++- 7 files changed, 558 insertions(+), 60 deletions(-) create mode 100644 sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 20fbb13e4..b7ec6371b 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -8,8 +8,14 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; import com.google.gson.annotations.JsonAdapter; import com.google.gson.annotations.SerializedName; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; @@ -30,6 +36,7 @@ import org.erdtman.jcs.JsonCanonicalizer; import java.io.IOException; +import java.io.StringReader; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -38,6 +45,7 @@ import java.security.interfaces.RSAPublicKey; import java.security.cert.X509Certificate; import java.text.ParseException; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -118,6 +126,192 @@ public int hashCode() { } } + /** + * A compact, append-only {@link Segment} list. + *

+ * A plain {@code ArrayList} costs roughly 176 bytes per segment, which + * is several gigabytes for a multi-terabyte payload. Every segment of a TDF but + * the last has the same sizes, and every segment hash is the same fixed length, + * so all this stores is the hash characters packed into fixed-stride chunks + * plus the two size pairs — about 24 bytes per segment, with no array-growth + * copies and no {@code Integer.MAX_VALUE} ceiling on the backing store. + *

+ * {@link #append} refuses anything that does not fit those assumptions (a + * manifest from another writer may not honor them); callers fall back to an + * {@code ArrayList}. + *

+ * ponytail: the remaining ceiling is {@link #aggregate}, which materializes one + * byte array of {@code count * hashLength} — about 134M segments (256 TiB at + * 2 MiB segments), or ~100M if the TDF carries assertions, whose signature is + * base64 of that array. Going past that needs a streaming Mac and streaming + * base64, and a wire-format change for assertions. + */ + static final class Segments extends AbstractList { + private static final int SEGMENTS_PER_CHUNK = 4096; + + private final List chunks = new ArrayList<>(); + private int count; + /** Length of every hash, in characters; established by the first append. */ + private int stride; + /** Trailing '=' in every hash, so decoded hashes are also fixed length. */ + private int padding; + private long defaultSegmentSize; + private long defaultEncryptedSegmentSize; + private long lastSegmentSize; + private long lastEncryptedSegmentSize; + + /** + * @return false if this segment cannot be represented compactly, in which + * case the segment was not appended and the caller must fall back + */ + boolean append(String hash, long segmentSize, long encryptedSegmentSize) { + if (hash == null) { + return false; + } + if (count == 0) { + stride = hash.length(); + padding = trailingPadding(hash); + defaultSegmentSize = segmentSize; + defaultEncryptedSegmentSize = encryptedSegmentSize; + } else if (hash.length() != stride + || trailingPadding(hash) != padding + // the previously appended segment is no longer the last one, so it + // has to match the defaults from here on + || lastSegmentSize != defaultSegmentSize + || lastEncryptedSegmentSize != defaultEncryptedSegmentSize) { + return false; + } + + if (count % SEGMENTS_PER_CHUNK == 0) { + chunks.add(new byte[stride * SEGMENTS_PER_CHUNK]); + } + byte[] chunk = chunks.get(count / SEGMENTS_PER_CHUNK); + int offset = (count % SEGMENTS_PER_CHUNK) * stride; + for (int i = 0; i < stride; i++) { + char c = hash.charAt(i); + if (c > 0x7f) { // non-ASCII would not survive the byte-per-character packing + return false; + } + chunk[offset + i] = (byte) c; + } + + lastSegmentSize = segmentSize; + lastEncryptedSegmentSize = encryptedSegmentSize; + count++; + return true; + } + + private static int trailingPadding(String hash) { + int padding = 0; + while (padding < hash.length() && hash.charAt(hash.length() - 1 - padding) == '=') { + padding++; + } + return padding; + } + + /** + * Every segment hash concatenated, base64-decoded if {@code decodeBase64} + * and taken as raw ASCII otherwise. This is the aggregate hash that the root + * signature and the assertion signatures are computed over. + */ + byte[] aggregate(boolean decodeBase64) { + if (count == 0) { + return new byte[0]; + } + byte[] scratch = new byte[stride]; + if (!decodeBase64) { + byte[] aggregate = new byte[Math.multiplyExact(count, stride)]; + for (int i = 0; i < count; i++) { + copyHash(i, aggregate, i * stride); + } + return aggregate; + } + + copyHash(0, scratch, 0); + int decodedStride = Base64.getDecoder().decode(scratch).length; + byte[] aggregate = new byte[Math.multiplyExact(count, decodedStride)]; + byte[] decoded = new byte[decodedStride]; + for (int i = 0; i < count; i++) { + copyHash(i, scratch, 0); + Base64.getDecoder().decode(scratch, decoded); + System.arraycopy(decoded, 0, aggregate, i * decodedStride, decodedStride); + } + return aggregate; + } + + private void copyHash(int index, byte[] destination, int destinationOffset) { + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + System.arraycopy(chunk, (index % SEGMENTS_PER_CHUNK) * stride, destination, destinationOffset, stride); + } + + @Override + public Segment get(int index) { + Objects.checkIndex(index, count); + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + Segment segment = new Segment(); + segment.hash = new String(chunk, (index % SEGMENTS_PER_CHUNK) * stride, stride, StandardCharsets.US_ASCII); + boolean last = index == count - 1; + segment.segmentSize = last ? lastSegmentSize : defaultSegmentSize; + segment.encryptedSegmentSize = last ? lastEncryptedSegmentSize : defaultEncryptedSegmentSize; + return segment; + } + + @Override + public int size() { + return count; + } + } + + /** + * Reads {@code segments} into a {@link Segments} when the manifest allows it, + * and into a plain {@code ArrayList} when it does not. Writing delegates to the + * reflective {@link Segment} adapter, so the JSON is unchanged either way. + */ + static final class SegmentsAdapterFactory implements TypeAdapterFactory { + @Override + @SuppressWarnings("unchecked") + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter element = gson.getAdapter(Segment.class); + return (TypeAdapter) new TypeAdapter>() { + @Override + public void write(JsonWriter out, List value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + out.beginArray(); + for (Segment segment : value) { + element.write(out, segment); + } + out.endArray(); + } + + @Override + public List read(JsonReader in) throws IOException { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return null; + } + in.beginArray(); + Segments compact = new Segments(); + List fallback = null; + while (in.hasNext()) { + Segment segment = element.read(in); + if (fallback != null) { + fallback.add(segment); + } else if (segment == null + || !compact.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)) { + fallback = new ArrayList<>(compact); + fallback.add(segment); + } + } + in.endArray(); + return fallback == null ? compact : fallback; + } + }; + } + } + static public class RootSignature { @SerializedName(value = "alg") public String algorithm; @@ -145,6 +339,7 @@ static public class IntegrityInformation { public String segmentHashAlg; public int segmentSizeDefault; public int encryptedSegmentSizeDefault; + @JsonAdapter(SegmentsAdapterFactory.class) public List segments; @Override @@ -559,6 +754,14 @@ public AssertionConfig.Statement deserialize(JsonElement json, Type typeOfT, Jso public Payload payload; public List assertions = new ArrayList<>(); protected static Manifest readManifest(String manifestJson) { + return readManifest(new StringReader(manifestJson)); + } + + /** + * Parses a manifest without materializing it as a {@link String}, which a + * manifest with tens of millions of segments cannot be. + */ + protected static Manifest readManifest(java.io.Reader manifestJson) { Manifest result = gson.fromJson(manifestJson, Manifest.class); if (result.assertions == null) { result.assertions = new ArrayList<>(); @@ -583,9 +786,13 @@ protected static Manifest readManifest(String manifestJson) { throw new IllegalArgumentException("Manifest with null policy"); } - for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { - if (segment == null || segment.hash == null) { - throw new IllegalArgumentException("Invalid integrity segment"); + // Segments rejects null segments and hashes as it is built, so only the + // fallback representation needs checking here. + if (!(result.encryptionInformation.integrityInformation.segments instanceof Segments)) { + for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { + if (segment == null || segment.hash == null) { + throw new IllegalArgumentException("Invalid integrity segment"); + } } } for (Manifest.KeyAccess keyAccess : result.encryptionInformation.keyAccessObj) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index 5e903498d..272bf8bed 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -185,8 +185,9 @@ public static boolean isTDF(SeekableByteChannel channel) { */ public static Manifest readManifest(SeekableByteChannel tdfBytes) throws SDKException, IOException { TDFReader reader = new TDFReader(tdfBytes); - String manifestJson = reader.manifest(); - return Manifest.readManifest(manifestJson); + try (var manifestJson = reader.manifest()) { + return Manifest.readManifest(manifestJson); + } } /** diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index b54901b98..775ae8193 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -15,10 +15,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.security.*; @@ -81,6 +83,13 @@ private static byte[] tdfECKeySaltCompute() { private static final Gson gson = new GsonBuilder().create(); + /** + * The manifest is serialized directly into the zip entry, and + * {@link ZipWriter#stream} updates its CRC on every {@code write} call, so the + * character encoder needs a buffer in front of it. + */ + private static final int MANIFEST_WRITE_BUFFER_SIZE = 1 << 16; + /** * A self-imposed ceiling on the number of AES-GCM authenticated-encryption * invocations under a single payload key. One invocation is spent on the @@ -472,6 +481,23 @@ public PolicyObject readPolicyObject() { } } + /** + * Concatenates every segment hash, which is what the root signature and the + * assertion signatures are computed over. + */ + private static byte[] aggregateSegmentHashes(List segments, boolean isEncrypted) { + if (segments instanceof Manifest.Segments) { + return ((Manifest.Segments) segments).aggregate(isEncrypted); + } + var aggregateHash = new ByteArrayOutputStream(); + for (Manifest.Segment segment : segments) { + aggregateHash.writeBytes(isEncrypted + ? Base64.getDecoder().decode(segment.hash) + : segment.hash.getBytes()); + } + return aggregateHash.toByteArray(); + } + private static byte[] calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) { if (algorithm == Config.IntegrityAlgorithm.HS256) { return CryptoUtils.CalculateSHA256Hmac(secret, data); @@ -501,11 +527,11 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo long encryptedSegmentSize = (long) tdfConfig.defaultSegmentSize + kGcmIvSize + AesGcm.GCM_TAG_LENGTH; TDFWriter tdfWriter = new TDFWriter(outputStream); - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); byte[] readBuf = new byte[tdfConfig.defaultSegmentSize]; IvCounter payloadIv = IvCounter.forPayload(); - tdfObject.manifest.encryptionInformation.integrityInformation.segments = new ArrayList<>(); + var segments = new Manifest.Segments(); + tdfObject.manifest.encryptionInformation.integrityInformation.segments = segments; boolean finished; try (var payloadOutput = tdfWriter.payload()) { do { @@ -519,7 +545,6 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo byte[] cipherData; byte[] segmentSig; - Manifest.Segment segmentInfo = new Manifest.Segment(); // encrypt cipherData = tdfObject.aesGcm.encrypt(payloadIv.next(), AesGcm.GCM_TAG_LENGTH, @@ -530,19 +555,23 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo if (tdfConfig.hexEncodeRootAndSegmentHashes) { segmentSig = Hex.encodeHexString(segmentSig).getBytes(StandardCharsets.UTF_8); } - segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig); - aggregateHash.write(segmentSig); - segmentInfo.segmentSize = readThisLoop; - segmentInfo.encryptedSegmentSize = cipherData.length; - - tdfObject.manifest.encryptionInformation.integrityInformation.segments.add(segmentInfo); + // Every segment we write is full except the last one, and every hash has the + // same length, so the compact representation always accepts them. + if (!segments.append(Base64.getEncoder().encodeToString(segmentSig), readThisLoop, + cipherData.length)) { + throw new SDKException("unable to record segment " + segments.size() + " in the manifest"); + } } while (!finished); } + // Materialized once and reused by the root signature and every assertion below; + // it is proportional to the number of segments. + byte[] aggregateHash = segments.aggregate(true); + Manifest.RootSignature rootSignature = new Manifest.RootSignature(); - byte[] rootSig = calculateSignature(aggregateHash.toByteArray(), tdfObject.payloadKey, + byte[] rootSig = calculateSignature(aggregateHash, tdfObject.payloadKey, tdfConfig.integrityAlgorithm); byte[] encodedRootSig = tdfConfig.hexEncodeRootAndSegmentHashes ? Hex.encodeHexString(rootSig).getBytes(StandardCharsets.UTF_8) @@ -595,9 +624,9 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo throw new SDKException("error decoding assertion hash", e); } } - byte[] completeHash = new byte[aggregateHash.size() + assertionHash.length]; - System.arraycopy(aggregateHash.toByteArray(), 0, completeHash, 0, aggregateHash.size()); - System.arraycopy(assertionHash, 0, completeHash, aggregateHash.size(), assertionHash.length); + byte[] completeHash = new byte[aggregateHash.length + assertionHash.length]; + System.arraycopy(aggregateHash, 0, completeHash, 0, aggregateHash.length); + System.arraycopy(assertionHash, 0, completeHash, aggregateHash.length, assertionHash.length); var encodedHash = Base64.getEncoder().encodeToString(completeHash); @@ -618,9 +647,13 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo } tdfObject.manifest.assertions = signedAssertions; - String manifestAsStr = gson.toJson(tdfObject.manifest); - tdfWriter.appendManifest(manifestAsStr); + // Serialize straight into the zip entry: a manifest with tens of millions of segments + // exceeds the maximum size of a Java String. + try (var manifestOutput = new BufferedWriter( + new OutputStreamWriter(tdfWriter.manifest(), StandardCharsets.UTF_8), MANIFEST_WRITE_BUFFER_SIZE)) { + gson.toJson(tdfObject.manifest, manifestOutput); + } tdfObject.size = tdfWriter.finish(); return tdfObject; @@ -657,9 +690,11 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig, Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throws SDKException, IOException { TDFReader tdfReader = new TDFReader(tdf); - String manifestJson = tdfReader.manifest(); // use Manifest.readManifest in order to validate the Manifest input - Manifest manifest = Manifest.readManifest(manifestJson); + Manifest manifest; + try (var manifestJson = tdfReader.manifest()) { + manifest = Manifest.readManifest(manifestJson); + } byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; @@ -744,15 +779,8 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) String rootAlgorithm = manifest.encryptionInformation.integrityInformation.rootSignature.algorithm; String rootSignature = manifest.encryptionInformation.integrityInformation.rootSignature.signature; - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - if (manifest.payload.isEncrypted) { - byte[] decodedHash = Base64.getDecoder().decode(segment.hash); - aggregateHash.write(decodedHash); - } else { - aggregateHash.write(segment.hash.getBytes()); - } - } + byte[] aggregateHash = aggregateSegmentHashes( + manifest.encryptionInformation.integrityInformation.segments, manifest.payload.isEncrypted); String rootSigValue; boolean isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); @@ -762,7 +790,7 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) sigAlg = Config.IntegrityAlgorithm.GMAC; } - var sig = calculateSignature(aggregateHash.toByteArray(), payloadKey, sigAlg); + var sig = calculateSignature(aggregateHash, payloadKey, sigAlg); if (isLegacyTdf) { sig = Hex.encodeHexString(sig).getBytes(); } @@ -775,7 +803,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new IllegalStateException("error getting instance of SHA-256 digest", e); } - rootSigValue = Base64.getEncoder().encodeToString(digest.digest(aggregateHash.toString().getBytes())); + // the round trip through the platform default charset is a no-op for the + // hex hashes this branch sees, and is kept so the digest is unchanged + rootSigValue = Base64.getEncoder().encodeToString(digest.digest(new String(aggregateHash).getBytes())); } if (rootSignature.compareTo(rootSigValue) != 0) { @@ -790,7 +820,6 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) "segment size mismatch. encrypted segment size differs from plaintext segment size. the TDF is invalid"); } - var aggregateHashByteArrayBytes = aggregateHash.toByteArray(); // Validate assertions for (var assertion : manifest.assertions) { // Skip assertion verification if disabled @@ -830,9 +859,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new SDKException("error decoding assertion hash", e); } } - var signature = new byte[aggregateHashByteArrayBytes.length + hashOfAssertion.length]; - System.arraycopy(aggregateHashByteArrayBytes, 0, signature, 0, aggregateHashByteArrayBytes.length); - System.arraycopy(hashOfAssertion, 0, signature, aggregateHashByteArrayBytes.length, hashOfAssertion.length); + var signature = new byte[aggregateHash.length + hashOfAssertion.length]; + System.arraycopy(aggregateHash, 0, signature, 0, aggregateHash.length); + System.arraycopy(hashOfAssertion, 0, signature, aggregateHash.length, hashOfAssertion.length); var encodeSignature = Base64.getEncoder().encodeToString(signature); if (!Objects.equals(encodeSignature, hashValues.getSignature())) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 6e9f32d21..ca068e45a 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -1,8 +1,10 @@ package io.opentdf.platform.sdk; -import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -18,6 +20,8 @@ */ public class TDFReader { + private static final int MANIFEST_BUFFER_SIZE = 1 << 16; + private final ZipReader.Entry manifestEntry; private final InputStream payload; @@ -37,15 +41,18 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); } - String manifest() { - var out = new ByteArrayOutputStream(); + /** + * The manifest entry as a character stream; the caller must close it. Returned + * as a stream rather than a String because a manifest with tens of millions of + * segments exceeds the maximum size of a Java String. + */ + Reader manifest() { try { - manifestEntry.getData().transferTo(out); + return new BufferedReader( + new InputStreamReader(manifestEntry.getData(), StandardCharsets.UTF_8), MANIFEST_BUFFER_SIZE); } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } - - return out.toString(StandardCharsets.UTF_8); } int readPayloadBytes(byte[] buf) { @@ -62,8 +69,10 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - String manifestJson = manifest(); - Manifest manifest = Manifest.readManifest(manifestJson); - return Manifest.decodePolicyObject(manifest); + try (Reader manifestJson = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestJson)); + } catch (IOException e) { + throw new SDKException("error reading manifest from zip file", e); + } } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 7137c2322..06357c2bb 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; /** * The TDFWriter class provides functionalities for creating a TDF (Trusted Data Format) archive. @@ -24,8 +23,12 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } - public void appendManifest(String manifest) throws IOException { - this.archiveWriter.data(TDF_MANIFEST_FILE_NAME, manifest.getBytes(StandardCharsets.UTF_8)); + /** + * Opens the manifest entry for writing. The returned stream must be closed before + * {@link #finish()} is called, otherwise the entry never makes it into the central directory. + */ + public OutputStream manifest() throws IOException { + return this.archiveWriter.stream(TDF_MANIFEST_FILE_NAME); } public OutputStream payload() throws IOException { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java new file mode 100644 index 000000000..ef80ce722 --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -0,0 +1,213 @@ +package io.opentdf.platform.sdk; + +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Covers the compact {@link Manifest.Segments} representation: it has to + * serialize exactly like a plain list, fall back when a manifest does not fit + * its assumptions, and let a manifest larger than a Java String round trip. + */ +public class SegmentsTest { + + private static final Gson GSON = new Gson(); + private static final int SEGMENT_SIZE = 16384; + private static final int ENCRYPTED_SEGMENT_SIZE = SEGMENT_SIZE + 28; + + private static String hash(int i) { + var bytes = new byte[16]; + bytes[0] = (byte) i; + bytes[1] = (byte) (i >> 8); + bytes[2] = (byte) (i >> 16); + return Base64.getEncoder().encodeToString(bytes); + } + + private static Manifest.Segment segment(String hash, long segmentSize, long encryptedSegmentSize) { + var segment = new Manifest.Segment(); + segment.hash = hash; + segment.segmentSize = segmentSize; + segment.encryptedSegmentSize = encryptedSegmentSize; + return segment; + } + + private static String manifestJson(List segments) { + return "{\"schemaVersion\":\"4.3.0\",\"encryptionInformation\":{\"type\":\"split\"," + // no '=' padding: Gson HTML-escapes it, and this literal is compared verbatim + + "\"policy\":\"eyJib2R5Ijp7fX0K\",\"keyAccess\":[{\"type\":\"wrapped\",\"url\":\"http://kas\"," + + "\"protocol\":\"kas\",\"wrappedKey\":\"AAAA\"}]," + + "\"method\":{\"algorithm\":\"AES-256-GCM\",\"iv\":\"AAAAAAAAAAAA\",\"isStreamable\":true}," + + "\"integrityInformation\":{\"rootSignature\":{\"alg\":\"GMAC\",\"sig\":\"AAAA\"}," + + "\"segmentHashAlg\":\"GMAC\",\"segmentSizeDefault\":" + SEGMENT_SIZE + + ",\"encryptedSegmentSizeDefault\":" + ENCRYPTED_SEGMENT_SIZE + ",\"segments\":" + + segments.stream().map(GSON::toJson).collect(Collectors.joining(",", "[", "]")) + + "}},\"payload\":{\"type\":\"reference\",\"url\":\"0.payload\",\"protocol\":\"zip\"," + + "\"mimeType\":\"application/octet-stream\",\"isEncrypted\":true},\"assertions\":[]}"; + } + + /** A run of uniform segments with a short final one, i.e. what this SDK writes. */ + private static List uniformSegments(int count) { + return IntStream.range(0, count) + .mapToObj(i -> segment(hash(i), + i == count - 1 ? 7 : SEGMENT_SIZE, + i == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)) + .collect(Collectors.toList()); + } + + @Test + void uniformSegmentsAreStoredCompactlyAndSerializeUnchanged() { + var segments = uniformSegments(5); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void irregularSegmentSizesFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + // a non-final segment that disagrees with the defaults cannot be stored compactly + segments.set(1, segment(hash(1), SEGMENT_SIZE / 2, ENCRYPTED_SEGMENT_SIZE / 2)); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void segmentsWithDifferingHashLengthsFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + segments.set(1, segment("deadbeef", SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + var json = manifestJson(segments); + + var parsed = Manifest.readManifest(json).encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + + @Test + void aggregateConcatenatesEveryHash() { + var segments = new Manifest.Segments(); + for (int i = 0; i < 3; i++) { + assertThat(segments.append(hash(i), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)).isTrue(); + } + + var decoded = new byte[48]; + for (int i = 0; i < 3; i++) { + System.arraycopy(Base64.getDecoder().decode(hash(i)), 0, decoded, i * 16, 16); + } + assertThat(segments.aggregate(true)).isEqualTo(decoded); + assertThat(segments.aggregate(false)) + .isEqualTo((hash(0) + hash(1) + hash(2)).getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + @Test + void emptySegmentsAggregateToNothing() { + assertThat(new Manifest.Segments().aggregate(true)).isEmpty(); + assertThat(new Manifest.Segments().aggregate(false)).isEmpty(); + } + + /** + * A manifest whose JSON is larger than {@code Integer.MAX_VALUE} cannot be held + * as a String, so both directions have to stream. Opt in with + * {@code -Dtdf.hugeSegments=true -Xmx2g}; it needs roughly a minute and 1 GiB. + */ + @Test + @EnabledIfSystemProperty(named = "tdf.hugeSegments", matches = "true") + void aManifestTooLargeToBeAStringRoundTrips() throws IOException { + // ~25M segments at ~99 JSON bytes each is ~2.5 GB, past the String ceiling + final int count = 25_000_000; + + Manifest manifest; + try (Reader json = new GeneratedManifestReader(count)) { + manifest = Manifest.readManifest(json); + } + var segments = manifest.encryptionInformation.integrityInformation.segments; + assertThat(segments).isInstanceOf(Manifest.Segments.class); + assertThat(segments.size()).isEqualTo(count); + assertThat(segments.get(count - 1).hash).isEqualTo(hash(count - 1)); + + var counting = new CountingWriter(); + new Gson().toJson(manifest, counting); + assertThat(counting.written).isGreaterThan(Integer.MAX_VALUE); + } + + /** Renders a manifest with {@code count} segments without ever storing it. */ + private static final class GeneratedManifestReader extends Reader { + private final int count; + private final String suffix; + private int next; + private String pending; + private int pendingOffset; + + GeneratedManifestReader(int count) { + this.count = count; + var template = manifestJson(uniformSegments(1)); + this.pending = template.substring(0, template.indexOf("\"segments\":[") + "\"segments\":[".length()); + this.suffix = template.substring(template.indexOf("]}},\"payload\"")); + } + + @Override + public int read(char[] buffer, int offset, int length) { + if (pendingOffset == pending.length()) { + if (next > count) { + return -1; + } + pending = next == count ? suffix + : (next == 0 ? "" : ",") + GSON.toJson(segment(hash(next), + next == count - 1 ? 7 : SEGMENT_SIZE, + next == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)); + pendingOffset = 0; + next++; + if (pending.isEmpty()) { + return read(buffer, offset, length); + } + } + int n = Math.min(length, pending.length() - pendingOffset); + pending.getChars(pendingOffset, pendingOffset + n, buffer, offset); + pendingOffset += n; + return n; + } + + @Override + public void close() { + } + } + + private static final class CountingWriter extends Writer { + private long written; + + @Override + public void write(char[] buffer, int offset, int length) { + written += length; + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + } +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java index 2f5d22fc5..84433a880 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java @@ -5,13 +5,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.StringWriter; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class TDFWriterTest { @Test @@ -64,14 +69,39 @@ void simpleTDFCreate() throws IOException { " }\n" + "}"; String payload = "Hello, world!"; - FileOutputStream fileOutStream = new FileOutputStream("sample.tdf"); - TDFWriter writer = new TDFWriter(fileOutStream); - try (var p = writer.payload()) { - new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + var tdfFile = File.createTempFile("sample", ".tdf"); + tdfFile.deleteOnExit(); + try (var fileOutStream = new FileOutputStream(tdfFile)) { + TDFWriter writer = new TDFWriter(fileOutStream); + try (var p = writer.payload()) { + new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + } + try (var m = writer.manifest()) { + m.write(kManifestJsonFromTDF.getBytes(StandardCharsets.UTF_8)); + } + writer.finish(); + } + + // our own reader + try (var channel = FileChannel.open(tdfFile.toPath(), StandardOpenOption.READ)) { + var entries = new ZipReader(channel).getEntries().stream() + .collect(Collectors.toMap(ZipReader.Entry::getName, e -> e)); + assertEquals(kManifestJsonFromTDF, + new String(entries.get(TDFWriter.TDF_MANIFEST_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + assertEquals(payload, + new String(entries.get(TDFWriter.TDF_PAYLOAD_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + } + + // an independent central-directory based reader, as a stand-in for the other SDKs + try (var zipFile = new ZipFile(tdfFile)) { + var manifestEntry = zipFile.getEntry(TDFWriter.TDF_MANIFEST_FILE_NAME); + assertNotNull(manifestEntry); + try (var in = zipFile.getInputStream(manifestEntry)) { + assertEquals(kManifestJsonFromTDF, new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } } - writer.appendManifest(kManifestJsonFromTDF); - writer.finish(); - fileOutStream.close(); } /** @@ -89,12 +119,18 @@ void readsBackAManifestWrittenPastTheZip64Boundary() throws IOException { try (var p = writer.payload()) { new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); } - writer.appendManifest(manifest); + try (var m = writer.manifest()) { + m.write(manifest.getBytes(StandardCharsets.UTF_8)); + } writer.finish(); try (var chan = new SeekableInMemoryByteChannel(out.toByteArray())) { var reader = new TDFReader(chan); - assertEquals(manifest, reader.manifest()); + var readBack = new StringWriter(); + try (var m = reader.manifest()) { + m.transferTo(readBack); + } + assertEquals(manifest, readBack.toString()); var payloadBytes = new byte[payload.length()]; assertEquals(payload.length(), reader.readPayloadBytes(payloadBytes)); From 4d084123caf648f04015679f86f4a1b94d4dcd58 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Fri, 11 Sep 2026 13:31:50 -0400 Subject: [PATCH 2/5] fix(sdk): bound segment hash stride, keep appendManifest as deprecated --- .../java/io/opentdf/platform/sdk/Manifest.java | 11 +++++++++++ .../java/io/opentdf/platform/sdk/TDFReader.java | 4 ++-- .../java/io/opentdf/platform/sdk/TDFWriter.java | 13 +++++++++++++ .../java/io/opentdf/platform/sdk/SegmentsTest.java | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index b7ec6371b..1a3de467f 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -148,6 +148,14 @@ public int hashCode() { */ static final class Segments extends AbstractList { private static final int SEGMENTS_PER_CHUNK = 4096; + /** + * A chunk is {@code stride * SEGMENTS_PER_CHUNK} bytes, so an absurd hash in an + * untrusted manifest would size the very first allocation. The longest hash any + * writer produces is 128 characters (hex-encoded HS256); anything past this is + * left to the {@code ArrayList} fallback, which only ever costs what the input + * itself costs. + */ + private static final int MAX_HASH_LENGTH = 1024; private final List chunks = new ArrayList<>(); private int count; @@ -169,6 +177,9 @@ boolean append(String hash, long segmentSize, long encryptedSegmentSize) { return false; } if (count == 0) { + if (hash.isEmpty() || hash.length() > MAX_HASH_LENGTH) { + return false; + } stride = hash.length(); padding = trailingPadding(hash); defaultSegmentSize = segmentSize; diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index ca068e45a..590a17c32 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -69,8 +69,8 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - try (Reader manifestJson = manifest()) { - return Manifest.decodePolicyObject(Manifest.readManifest(manifestJson)); + try (Reader manifestRaw = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestRaw)); } catch (IOException e) { throw new SDKException("error reading manifest from zip file", e); } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 06357c2bb..b64a39f13 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; /** * The TDFWriter class provides functionalities for creating a TDF (Trusted Data Format) archive. @@ -23,6 +24,18 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } + /** + * @deprecated use {@link #manifest()}. A manifest with tens of millions of segments + * exceeds the maximum size of a Java {@link String}, so this cannot + * express every manifest the SDK writes. It produces the same zip entry. + */ + @Deprecated + public void appendManifest(String manifest) throws IOException { + try (OutputStream output = manifest()) { + output.write(manifest.getBytes(StandardCharsets.UTF_8)); + } + } + /** * Opens the manifest entry for writing. The returned stream must be closed before * {@link #finish()} is called, otherwise the entry never makes it into the central directory. diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java index ef80ce722..3c5155ee1 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -105,6 +105,20 @@ void segmentsWithDifferingHashLengthsFallBackToAPlainList() { assertThat(parsed).containsExactlyElementsOf(segments); } + @Test + void anAbsurdlyLongHashFallsBackInsteadOfSizingAChunkFromIt() { + // a chunk is stride * 4096 bytes, so without a bound this one hash would ask for + // gigabytes before the fallback ever ran + var segments = new ArrayList<>(uniformSegments(2)); + segments.set(0, segment("A".repeat(100_000), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + + var parsed = Manifest.readManifest(manifestJson(segments)) + .encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + @Test void aggregateConcatenatesEveryHash() { var segments = new Manifest.Segments(); From 2449cdb5067b7d9b31a7ccfff4192661af1496c1 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Wed, 21 Oct 2026 15:12:50 -0400 Subject: [PATCH 3/5] fix(sdk): address manifest streaming review findings Signed-off-by: sujan kota --- .../io/opentdf/platform/sdk/Manifest.java | 252 +++++++++++++----- .../java/io/opentdf/platform/sdk/TDF.java | 6 +- .../io/opentdf/platform/sdk/ManifestTest.java | 21 ++ .../io/opentdf/platform/sdk/SegmentsTest.java | 25 ++ 4 files changed, 239 insertions(+), 65 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 859a9dbe3..2af7e41b3 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -1,5 +1,7 @@ package io.opentdf.platform.sdk; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializationContext; @@ -36,8 +38,13 @@ import org.apache.commons.codec.binary.Hex; import org.erdtman.jcs.JsonCanonicalizer; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -49,7 +56,9 @@ import java.util.AbstractList; import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -64,6 +73,7 @@ public class Manifest { private static final String kAssertionSignature = "assertionSig"; private static final Gson gson = new GsonBuilder() + .addSerializationExclusionStrategy(new DeserializeOnlyExclusionStrategy()) .registerTypeAdapter(AssertionConfig.Statement.class, new AssertionValueAdapter()) .registerTypeAdapterFactory(new IntegrityInformationAdapterFactory()) .registerTypeAdapterFactory(new SpecVersionAdapterFactory()) @@ -80,6 +90,27 @@ public class Manifest { @SerializedName(value = "schemaVersion") String tdfVersion; + @DeserializeOnly + @SerializedName("tdf_spec_version") + JsonElement nonAlignedSpecVersion; + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.FIELD) + private @interface DeserializeOnly { + } + + private static final class DeserializeOnlyExclusionStrategy implements ExclusionStrategy { + @Override + public boolean shouldSkipField(FieldAttributes field) { + return field.getAnnotation(DeserializeOnly.class) != null; + } + + @Override + public boolean shouldSkipClass(Class type) { + return false; + } + } + public static String toJson(Manifest manifest) { return gson.toJson(manifest); } @@ -131,7 +162,9 @@ static public class Segment { * {@link IntegrityInformation#encryptedSegmentSizeDefault}. */ public long encryptedSegmentSize; + @SuppressWarnings("java:S2065") // Gson uses transient to keep parser state out of manifest JSON. private transient boolean segmentSizePresent = true; + @SuppressWarnings("java:S2065") // Gson uses transient to keep parser state out of manifest JSON. private transient boolean encryptedSegmentSizePresent = true; @Override @@ -184,6 +217,8 @@ static final class Segments extends AbstractList { private static final int MAX_HASH_LENGTH = 1024; private final List chunks = new ArrayList<>(); + private final Map materializedSegments = new HashMap<>(); + private List mutableSegments; private int count; /** Length of every hash, in characters; established by the first append. */ private int stride; @@ -207,7 +242,7 @@ boolean append(String hash, long segmentSize, long encryptedSegmentSize) { } private boolean append(Segment segment) { - return append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize, + return segment != null && append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize, segment.segmentSizePresent, segment.encryptedSegmentSizePresent); } @@ -295,9 +330,12 @@ private static int trailingPadding(String hash) { * signature and the assertion signatures are computed over. */ byte[] aggregate(boolean decodeBase64) { - if (count == 0) { + if (size() == 0) { return new byte[0]; } + if (mutableSegments != null || !materializedSegments.isEmpty()) { + return aggregateMaterialized(decodeBase64); + } byte[] scratch = new byte[stride]; if (!decodeBase64) { byte[] aggregate = new byte[Math.multiplyExact(count, stride)]; @@ -319,6 +357,18 @@ byte[] aggregate(boolean decodeBase64) { return aggregate; } + private byte[] aggregateMaterialized(boolean decodeBase64) { + ByteArrayOutputStream aggregate = new ByteArrayOutputStream(); + for (int i = 0; i < size(); i++) { + String hash = getForRead(i).hash; + byte[] bytes = decodeBase64 + ? Base64.getDecoder().decode(hash) + : hash.getBytes(StandardCharsets.US_ASCII); + aggregate.write(bytes, 0, bytes.length); + } + return aggregate.toByteArray(); + } + private void copyHash(int index, byte[] destination, int destinationOffset) { byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); System.arraycopy(chunk, (index % SEGMENTS_PER_CHUNK) * stride, destination, destinationOffset, stride); @@ -326,7 +376,23 @@ private void copyHash(int index, byte[] destination, int destinationOffset) { @Override public Segment get(int index) { + if (mutableSegments != null) { + return mutableSegments.get(index); + } + Objects.checkIndex(index, count); + return materializedSegments.computeIfAbsent(index, this::unpack); + } + + Segment getForRead(int index) { + if (mutableSegments != null) { + return mutableSegments.get(index); + } Objects.checkIndex(index, count); + Segment materialized = materializedSegments.get(index); + return materialized == null ? unpack(index) : materialized; + } + + private Segment unpack(int index) { byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); Segment segment = new Segment(); segment.hash = new String(chunk, (index % SEGMENTS_PER_CHUNK) * stride, stride, StandardCharsets.US_ASCII); @@ -339,12 +405,54 @@ public Segment get(int index) { return segment; } + private void materialize() { + if (mutableSegments != null) { + return; + } + List copy = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + copy.add(getForRead(i)); + } + mutableSegments = copy; + materializedSegments.clear(); + chunks.clear(); + } + + @Override + public Segment set(int index, Segment element) { + materialize(); + return mutableSegments.set(index, element); + } + + @Override + public void add(int index, Segment element) { + materialize(); + mutableSegments.add(index, element); + count = mutableSegments.size(); + modCount++; + } + + @Override + public Segment remove(int index) { + materialize(); + Segment removed = mutableSegments.remove(index); + count = mutableSegments.size(); + modCount++; + return removed; + } + @Override public int size() { - return count; + return mutableSegments == null ? count : mutableSegments.size(); } } + static Segment segmentAt(List segments, int index) { + return segments instanceof Segments + ? ((Segments) segments).getForRead(index) + : segments.get(index); + } + /** * Reads {@code segments} into a {@link Segments} when the manifest allows it, * and into a plain {@code ArrayList} when it does not. Writing delegates to the @@ -356,50 +464,7 @@ static final class SegmentsAdapterFactory implements TypeAdapterFactory { public TypeAdapter create(Gson gson, TypeToken type) { TypeAdapter element = gson.getAdapter(Segment.class); TypeAdapter jsonElement = gson.getAdapter(JsonElement.class); - return (TypeAdapter) new TypeAdapter>() { - @Override - public void write(JsonWriter out, List value) throws IOException { - if (value == null) { - out.nullValue(); - return; - } - out.beginArray(); - for (Segment segment : value) { - element.write(out, segment); - } - out.endArray(); - } - - @Override - @SuppressWarnings("java:S1168") // Preserve JSON null so manifest validation rejects it. - public List read(JsonReader in) throws IOException { - if (in.peek() == JsonToken.NULL) { - in.nextNull(); - return null; - } - in.beginArray(); - Segments compact = new Segments(); - List fallback = null; - while (in.hasNext()) { - JsonElement rawSegment = jsonElement.read(in); - Segment segment = element.fromJsonTree(rawSegment); - if (segment != null && rawSegment.isJsonObject()) { - JsonObject object = rawSegment.getAsJsonObject(); - segment.segmentSizePresent = hasValue(object, "segmentSize"); - segment.encryptedSegmentSizePresent = hasValue(object, "encryptedSegmentSize"); - } - if (fallback != null) { - fallback.add(segment); - } else if (segment == null - || !compact.append(segment)) { - fallback = new ArrayList<>(compact); - fallback.add(segment); - } - } - in.endArray(); - return fallback == null ? compact : fallback; - } - }; + return (TypeAdapter) new SegmentsTypeAdapter(element, jsonElement); } private static boolean hasValue(JsonObject object, String memberName) { @@ -408,6 +473,65 @@ private static boolean hasValue(JsonObject object, String memberName) { } } + private static final class SegmentsTypeAdapter extends TypeAdapter> { + private final TypeAdapter elementAdapter; + private final TypeAdapter jsonElementAdapter; + + private SegmentsTypeAdapter(TypeAdapter elementAdapter, + TypeAdapter jsonElementAdapter) { + this.elementAdapter = elementAdapter; + this.jsonElementAdapter = jsonElementAdapter; + } + + @Override + public void write(JsonWriter out, List value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + out.beginArray(); + for (int i = 0; i < value.size(); i++) { + elementAdapter.write(out, segmentAt(value, i)); + } + out.endArray(); + } + + @Override + @SuppressWarnings("java:S1168") // Preserve JSON null so manifest validation rejects it. + public List read(JsonReader in) throws IOException { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return null; + } + in.beginArray(); + Segments compact = new Segments(); + List fallback = null; + while (in.hasNext()) { + Segment segment = readSegment(in); + if (fallback != null) { + fallback.add(segment); + } else if (!compact.append(segment)) { + fallback = new ArrayList<>(compact); + fallback.add(segment); + } + } + in.endArray(); + return fallback == null ? compact : fallback; + } + + private Segment readSegment(JsonReader in) throws IOException { + JsonElement rawSegment = jsonElementAdapter.read(in); + Segment segment = elementAdapter.fromJsonTree(rawSegment); + if (segment != null && rawSegment.isJsonObject()) { + JsonObject object = rawSegment.getAsJsonObject(); + segment.segmentSizePresent = SegmentsAdapterFactory.hasValue(object, "segmentSize"); + segment.encryptedSegmentSizePresent = SegmentsAdapterFactory.hasValue( + object, "encryptedSegmentSize"); + } + return segment; + } + } + static public class RootSignature { @SerializedName(value = "alg") public String algorithm; @@ -561,16 +685,12 @@ private static void applySegmentSizeDefaults(IntegrityInformation integrityInfor * document instead of preferring {@code schemaVersion}. */ private static class SpecVersionAdapterFactory implements TypeAdapterFactory { - private static final String NON_ALIGNED_SPEC_VERSION = "tdf_spec_version"; - private static final String PAYLOAD = "payload"; - @Override public TypeAdapter create(Gson gson, TypeToken type) { if (!Manifest.class.equals(type.getRawType())) { return null; } final TypeAdapter delegate = gson.getDelegateAdapter(this, type); - final TypeAdapter elementAdapter = gson.getAdapter(JsonElement.class); return new TypeAdapter() { @Override public void write(JsonWriter out, T value) throws IOException { @@ -579,12 +699,11 @@ public void write(JsonWriter out, T value) throws IOException { @Override public T read(JsonReader in) throws IOException { - JsonElement tree = elementAdapter.read(in); - T value = delegate.fromJsonTree(tree); - if (value instanceof Manifest && tree != null && tree.isJsonObject()) { + T value = delegate.read(in); + if (value instanceof Manifest) { Manifest manifest = (Manifest) value; if (manifest.tdfVersion == null || manifest.tdfVersion.isEmpty()) { - String nonAligned = nonAlignedSpecVersion(tree.getAsJsonObject()); + String nonAligned = nonAlignedSpecVersion(manifest); if (nonAligned != null) { manifest.tdfVersion = nonAligned; } @@ -596,16 +715,12 @@ public T read(JsonReader in) throws IOException { } /** The first non-empty string under the non-aligned name, root before payload, or null. */ - private static String nonAlignedSpecVersion(JsonObject root) { - String atRoot = nonEmptyString(root.get(NON_ALIGNED_SPEC_VERSION)); + private static String nonAlignedSpecVersion(Manifest manifest) { + String atRoot = nonEmptyString(manifest.nonAlignedSpecVersion); if (atRoot != null) { return atRoot; } - JsonElement payload = root.get(PAYLOAD); - if (payload != null && payload.isJsonObject()) { - return nonEmptyString(payload.getAsJsonObject().get(NON_ALIGNED_SPEC_VERSION)); - } - return null; + return manifest.payload == null ? null : nonEmptyString(manifest.payload.nonAlignedSpecVersion); } private static String nonEmptyString(JsonElement element) { @@ -729,6 +844,9 @@ static public class Payload { public String protocol; public String mimeType; public boolean isEncrypted; + @DeserializeOnly + @SerializedName("tdf_spec_version") + JsonElement nonAlignedSpecVersion; @Override public boolean equals(Object o) { @@ -1017,7 +1135,15 @@ protected static Manifest readManifest(String manifestJson) { * manifest with tens of millions of segments cannot be. */ protected static Manifest readManifest(java.io.Reader manifestJson) { - Manifest result = gson.fromJson(manifestJson, Manifest.class); + Manifest result; + try { + result = gson.fromJson(manifestJson, Manifest.class); + } catch (JsonParseException e) { + if (e.getCause() instanceof IOException) { + throw new SDKException("error reading manifest from zip file", (IOException) e.getCause()); + } + throw e; + } if (result == null) { throw new IllegalArgumentException("Manifest is empty"); } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 082b03bc9..883d885da 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -407,7 +407,9 @@ public void readPayload(OutputStream outputStream) throws SDK.TamperException, I validateSegmentSizes(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { + List segments = manifest.encryptionInformation.integrityInformation.segments; + for (int i = 0; i < segments.size(); i++) { + Manifest.Segment segment = Manifest.segmentAt(segments, i); byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; int bytesRead = tdfReader.readPayloadBytes(readBuf); @@ -462,7 +464,7 @@ private void validateSegmentSizes() { List segments = manifest.encryptionInformation.integrityInformation.segments; for (int i = 0; i < segments.size(); i++) { - long encryptedSegmentSize = segments.get(i).encryptedSegmentSize; + long encryptedSegmentSize = Manifest.segmentAt(segments, i).encryptedSegmentSize; if (encryptedSegmentSize < minEncryptedSegmentSize) { throw new IllegalStateException("invalid TDF: segment " + i diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java index 04070a614..63734f143 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.io.Reader; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -271,6 +272,26 @@ void rejectsKeyAccessWithoutUrl() { .hasMessage("Invalid KeyAccess with null url in manifest"); } + @Test + void wrapsManifestReaderIoFailures() { + Reader failingReader = new Reader() { + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + throw new IOException("manifest read failed"); + } + + @Override + public void close() { + // No resources to release. + } + }; + + assertThatThrownBy(() -> Manifest.readManifest(failingReader)) + .isInstanceOf(SDKException.class) + .hasMessage("error reading manifest from zip file") + .hasRootCauseMessage("manifest read failed"); + } + @Test void testReadingManifestWithObjectStatementValue() throws IOException { final Manifest manifest; diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java index e26d4f28b..ea4d60697 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -139,6 +139,31 @@ void aggregateConcatenatesEveryHash() { .isEqualTo((hash(0) + hash(1) + hash(2)).getBytes(java.nio.charset.StandardCharsets.US_ASCII)); } + @Test + void parsedCompactSegmentsPreserveElementAndListMutations() { + var manifest = Manifest.readManifest(manifestJson(uniformSegments(3))); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + assertThat(parsed).isInstanceOf(Manifest.Segments.class); + + Manifest.Segment first = parsed.get(0); + first.hash = hash(10); + first.segmentSize = 123; + + Manifest.Segment replacement = segment(hash(11), 456, 484); + Manifest.Segment replaced = parsed.set(1, replacement); + Manifest.Segment added = segment(hash(12), 789, 817); + parsed.add(added); + Manifest.Segment removed = parsed.remove(2); + + assertThat(replaced.hash).isEqualTo(hash(1)); + assertThat(removed.hash).isEqualTo(hash(2)); + assertThat(parsed).containsExactly(first, replacement, added); + + var roundTripped = Manifest.readManifest(Manifest.toJson(manifest)) + .encryptionInformation.integrityInformation.segments; + assertThat(roundTripped).containsExactlyElementsOf(parsed); + } + @Test void emptySegmentsAggregateToNothing() { assertThat(new Manifest.Segments().aggregate(true)).isEmpty(); From ff1edb2d8d128038257ca034219e5ea1ddafc76d Mon Sep 17 00:00:00 2001 From: sujan kota Date: Thu, 22 Oct 2026 10:13:01 -0400 Subject: [PATCH 4/5] fix(sdk): preserve compact manifest error semantics Signed-off-by: sujan kota --- .../io/opentdf/platform/sdk/Manifest.java | 37 ++++++++++++++++++- .../io/opentdf/platform/sdk/ManifestTest.java | 8 ++++ .../io/opentdf/platform/sdk/SegmentsTest.java | 15 ++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 2af7e41b3..854534c8e 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -19,6 +19,7 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; +import com.google.gson.stream.MalformedJsonException; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; @@ -39,6 +40,7 @@ import org.erdtman.jcs.JsonCanonicalizer; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.StringReader; import java.lang.annotation.ElementType; @@ -204,7 +206,6 @@ public int hashCode() { * base64 of that array. Going past that needs a streaming Mac and streaming * base64, and a wire-format change for assertions. */ - @SuppressWarnings("java:S2160") // AbstractList equality is the intended list-value equality. static final class Segments extends AbstractList { private static final int SEGMENTS_PER_CHUNK = 4096; /** @@ -445,6 +446,36 @@ public Segment remove(int index) { public int size() { return mutableSegments == null ? count : mutableSegments.size(); } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof List) || size() != ((List) other).size()) { + return false; + } + List otherList = (List) other; + for (int i = 0; i < size(); i++) { + Object otherSegment = otherList instanceof Segments + ? ((Segments) otherList).getForRead(i) + : otherList.get(i); + if (!Objects.equals(getForRead(i), otherSegment)) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + for (int i = 0; i < size(); i++) { + Segment segment = getForRead(i); + hashCode = 31 * hashCode + (segment == null ? 0 : segment.hashCode()); + } + return hashCode; + } } static Segment segmentAt(List segments, int index) { @@ -1139,7 +1170,9 @@ protected static Manifest readManifest(java.io.Reader manifestJson) { try { result = gson.fromJson(manifestJson, Manifest.class); } catch (JsonParseException e) { - if (e.getCause() instanceof IOException) { + if (e.getCause() instanceof IOException + && !(e.getCause() instanceof MalformedJsonException) + && !(e.getCause() instanceof EOFException)) { throw new SDKException("error reading manifest from zip file", (IOException) e.getCause()); } throw e; diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java index 63734f143..2ae3e3691 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java @@ -1,6 +1,7 @@ package io.opentdf.platform.sdk; import com.google.gson.Gson; +import com.google.gson.JsonParseException; import com.google.gson.JsonParser; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -292,6 +293,13 @@ public void close() { .hasRootCauseMessage("manifest read failed"); } + @Test + void preservesMalformedJsonErrors() { + assertThatThrownBy(() -> Manifest.readManifest("{\"payload\":")) + .isInstanceOf(JsonParseException.class) + .isNotInstanceOf(SDKException.class); + } + @Test void testReadingManifestWithObjectStatementValue() throws IOException { final Manifest manifest; diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java index ea4d60697..745110954 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -170,6 +170,21 @@ void emptySegmentsAggregateToNothing() { assertThat(new Manifest.Segments().aggregate(false)).isEmpty(); } + @Test + void compactSegmentsRetainListEqualityAndHashing() { + var left = new Manifest.Segments(); + var right = new Manifest.Segments(); + var expected = uniformSegments(3); + for (Manifest.Segment segment : expected) { + assertThat(left.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)).isTrue(); + assertThat(right.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)).isTrue(); + } + + assertThat(left).isEqualTo(right); + assertThat(left.hashCode()).isEqualTo(right.hashCode()); + assertThat(left).isEqualTo(new ArrayList<>(expected)); + } + /** * A manifest whose JSON is larger than {@code Integer.MAX_VALUE} cannot be held * as a String, so both directions have to stream. Opt in with From e88d3d9052fa940fef81815b8ae6c9af0f48802c Mon Sep 17 00:00:00 2001 From: sujan kota Date: Thu, 22 Oct 2026 11:29:34 -0400 Subject: [PATCH 5/5] fix(sdk): distinguish manifest source read failures Signed-off-by: sujan kota --- .../io/opentdf/platform/sdk/Manifest.java | 53 ++++++++++++++----- .../io/opentdf/platform/sdk/ManifestTest.java | 29 ++++++++++ .../io/opentdf/platform/sdk/SegmentsTest.java | 4 +- 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 854534c8e..e751a2120 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -19,7 +19,6 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; -import com.google.gson.stream.MalformedJsonException; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; @@ -40,8 +39,8 @@ import org.erdtman.jcs.JsonCanonicalizer; import java.io.ByteArrayOutputStream; -import java.io.EOFException; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -59,6 +58,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -456,11 +456,17 @@ public boolean equals(Object other) { return false; } List otherList = (List) other; + if (!(otherList instanceof Segments)) { + Iterator otherSegments = otherList.iterator(); + for (int i = 0; i < size(); i++) { + if (!Objects.equals(getForRead(i), otherSegments.next())) { + return false; + } + } + return true; + } for (int i = 0; i < size(); i++) { - Object otherSegment = otherList instanceof Segments - ? ((Segments) otherList).getForRead(i) - : otherList.get(i); - if (!Objects.equals(getForRead(i), otherSegment)) { + if (!Objects.equals(getForRead(i), ((Segments) otherList).getForRead(i))) { return false; } } @@ -1165,15 +1171,14 @@ protected static Manifest readManifest(String manifestJson) { * Parses a manifest without materializing it as a {@link String}, which a * manifest with tens of millions of segments cannot be. */ - protected static Manifest readManifest(java.io.Reader manifestJson) { + protected static Manifest readManifest(Reader manifestJson) { + FailureTrackingReader reader = new FailureTrackingReader(manifestJson); Manifest result; try { - result = gson.fromJson(manifestJson, Manifest.class); + result = gson.fromJson(reader, Manifest.class); } catch (JsonParseException e) { - if (e.getCause() instanceof IOException - && !(e.getCause() instanceof MalformedJsonException) - && !(e.getCause() instanceof EOFException)) { - throw new SDKException("error reading manifest from zip file", (IOException) e.getCause()); + if (reader.failure != null) { + throw new SDKException("error reading manifest from zip file", reader.failure); } throw e; } @@ -1184,6 +1189,30 @@ protected static Manifest readManifest(java.io.Reader manifestJson) { return result; } + private static final class FailureTrackingReader extends Reader { + private final Reader delegate; + private IOException failure; + + private FailureTrackingReader(Reader delegate) { + this.delegate = delegate; + } + + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + try { + return delegate.read(buffer, offset, length); + } catch (IOException e) { + failure = e; + throw e; + } + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } + private static void validateManifest(Manifest result) { if (result.assertions == null) { result.assertions = new ArrayList<>(); diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java index 2ae3e3691..c49a709df 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.io.EOFException; import java.io.Reader; import java.nio.charset.StandardCharsets; import java.util.List; @@ -300,6 +301,34 @@ void preservesMalformedJsonErrors() { .isNotInstanceOf(SDKException.class); } + @Test + void wrapsEofThrownByTheSourceReader() { + Reader failingReader = new Reader() { + private final String partialManifest = "{\"payload\":"; + private int position; + + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + if (position == partialManifest.length()) { + throw new EOFException("archive ended early"); + } + int copied = Math.min(length, partialManifest.length() - position); + partialManifest.getChars(position, position + copied, buffer, offset); + position += copied; + return copied; + } + + @Override + public void close() { + // No resources to release. + } + }; + + assertThatThrownBy(() -> Manifest.readManifest(failingReader)) + .isInstanceOf(SDKException.class) + .hasRootCauseMessage("archive ended early"); + } + @Test void testReadingManifestWithObjectStatementValue() throws IOException { final Manifest manifest; diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java index 745110954..9333d57c9 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -181,8 +181,8 @@ void compactSegmentsRetainListEqualityAndHashing() { } assertThat(left).isEqualTo(right); - assertThat(left.hashCode()).isEqualTo(right.hashCode()); - assertThat(left).isEqualTo(new ArrayList<>(expected)); + assertThat(left).hasSameHashCodeAs(right); + assertThat(left).containsExactlyElementsOf(expected); } /**