diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index ab1bee5f..e751a212 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -1,8 +1,9 @@ package io.opentdf.platform.sdk; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; @@ -16,6 +17,7 @@ import com.google.gson.annotations.SerializedName; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; @@ -36,7 +38,14 @@ import org.apache.commons.codec.binary.Hex; import org.erdtman.jcs.JsonCanonicalizer; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -45,9 +54,13 @@ import java.security.interfaces.RSAPublicKey; import java.security.cert.X509Certificate; import java.text.ParseException; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -62,6 +75,7 @@ public class Manifest { private static final String kAssertionSignature = "assertionSig"; private static final Gson gson = new GsonBuilder() + .addSerializationExclusionStrategy(new DeserializeOnlyExclusionStrategy()) .registerTypeAdapter(AssertionConfig.Statement.class, new AssertionValueAdapter()) .registerTypeAdapterFactory(new IntegrityInformationAdapterFactory()) .registerTypeAdapterFactory(new SpecVersionAdapterFactory()) @@ -78,6 +92,27 @@ public class Manifest { @SerializedName(value = "schemaVersion") String tdfVersion; + @DeserializeOnly + @SerializedName("tdf_spec_version") + JsonElement nonAlignedSpecVersion; + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.FIELD) + private @interface DeserializeOnly { + } + + private static final class DeserializeOnlyExclusionStrategy implements ExclusionStrategy { + @Override + public boolean shouldSkipField(FieldAttributes field) { + return field.getAnnotation(DeserializeOnly.class) != null; + } + + @Override + public boolean shouldSkipClass(Class type) { + return false; + } + } + public static String toJson(Manifest manifest) { return gson.toJson(manifest); } @@ -129,6 +164,10 @@ static public class Segment { * {@link IntegrityInformation#encryptedSegmentSizeDefault}. */ public long encryptedSegmentSize; + @SuppressWarnings("java:S2065") // Gson uses transient to keep parser state out of manifest JSON. + private transient boolean segmentSizePresent = true; + @SuppressWarnings("java:S2065") // Gson uses transient to keep parser state out of manifest JSON. + private transient boolean encryptedSegmentSizePresent = true; @Override public boolean equals(Object o) { @@ -147,6 +186,389 @@ public int hashCode() { } } + /** + * A compact, append-only {@link Segment} list. + *

+ * A plain {@code ArrayList} costs roughly 176 bytes per segment, which + * is several gigabytes for a multi-terabyte payload. Every segment of a TDF but + * the last has the same sizes, and every segment hash is the same fixed length, + * so all this stores is the hash characters packed into fixed-stride chunks + * plus the two size pairs — about 24 bytes per segment, with no array-growth + * copies and no {@code Integer.MAX_VALUE} ceiling on the backing store. + *

+ * {@link #append} refuses anything that does not fit those assumptions (a + * manifest from another writer may not honor them); callers fall back to an + * {@code ArrayList}. + *

+ * ponytail: the remaining ceiling is {@link #aggregate}, which materializes one + * byte array of {@code count * hashLength} — about 134M segments (256 TiB at + * 2 MiB segments), or ~100M if the TDF carries assertions, whose signature is + * base64 of that array. Going past that needs a streaming Mac and streaming + * base64, and a wire-format change for assertions. + */ + static final class Segments extends AbstractList { + private static final int SEGMENTS_PER_CHUNK = 4096; + /** + * A chunk is {@code stride * SEGMENTS_PER_CHUNK} bytes, so an absurd hash in an + * untrusted manifest would size the very first allocation. The longest hash any + * writer produces is 128 characters (hex-encoded HS256); anything past this is + * left to the {@code ArrayList} fallback, which only ever costs what the input + * itself costs. + */ + private static final int MAX_HASH_LENGTH = 1024; + + private final List chunks = new ArrayList<>(); + private final Map materializedSegments = new HashMap<>(); + private List mutableSegments; + private int count; + /** Length of every hash, in characters; established by the first append. */ + private int stride; + /** Trailing '=' in every hash, so decoded hashes are also fixed length. */ + private int padding; + private long defaultSegmentSize; + private long defaultEncryptedSegmentSize; + private boolean defaultSegmentSizePresent; + private boolean defaultEncryptedSegmentSizePresent; + private long lastSegmentSize; + private long lastEncryptedSegmentSize; + private boolean lastSegmentSizePresent; + private boolean lastEncryptedSegmentSizePresent; + + /** + * @return false if this segment cannot be represented compactly, in which + * case the segment was not appended and the caller must fall back + */ + boolean append(String hash, long segmentSize, long encryptedSegmentSize) { + return append(hash, segmentSize, encryptedSegmentSize, true, true); + } + + private boolean append(Segment segment) { + return segment != null && append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize, + segment.segmentSizePresent, segment.encryptedSegmentSizePresent); + } + + private boolean append(String hash, long segmentSize, long encryptedSegmentSize, + boolean segmentSizePresent, boolean encryptedSegmentSizePresent) { + if (hash == null) { + return false; + } + if (count == 0) { + if (hash.isEmpty() || hash.length() > MAX_HASH_LENGTH) { + return false; + } + stride = hash.length(); + padding = trailingPadding(hash); + defaultSegmentSize = segmentSize; + defaultEncryptedSegmentSize = encryptedSegmentSize; + defaultSegmentSizePresent = segmentSizePresent; + defaultEncryptedSegmentSizePresent = encryptedSegmentSizePresent; + } else if (hash.length() != stride + || trailingPadding(hash) != padding + // the previously appended segment is no longer the last one, so it + // has to match the defaults from here on + || !sameValue(lastSegmentSize, lastSegmentSizePresent, + defaultSegmentSize, defaultSegmentSizePresent) + || !sameValue(lastEncryptedSegmentSize, lastEncryptedSegmentSizePresent, + defaultEncryptedSegmentSize, defaultEncryptedSegmentSizePresent)) { + return false; + } + + if (count % SEGMENTS_PER_CHUNK == 0) { + chunks.add(new byte[stride * SEGMENTS_PER_CHUNK]); + } + byte[] chunk = chunks.get(count / SEGMENTS_PER_CHUNK); + int offset = (count % SEGMENTS_PER_CHUNK) * stride; + for (int i = 0; i < stride; i++) { + char c = hash.charAt(i); + if (c > 0x7f) { // non-ASCII would not survive the byte-per-character packing + return false; + } + chunk[offset + i] = (byte) c; + } + + lastSegmentSize = segmentSize; + lastEncryptedSegmentSize = encryptedSegmentSize; + lastSegmentSizePresent = segmentSizePresent; + lastEncryptedSegmentSizePresent = encryptedSegmentSizePresent; + count++; + return true; + } + + private static boolean sameValue(long left, boolean leftPresent, long right, boolean rightPresent) { + return leftPresent == rightPresent && (!leftPresent || left == right); + } + + private void applyDefaults(long segmentSize, long encryptedSegmentSize) { + if (!defaultSegmentSizePresent) { + defaultSegmentSize = segmentSize; + defaultSegmentSizePresent = true; + } + if (!defaultEncryptedSegmentSizePresent) { + defaultEncryptedSegmentSize = encryptedSegmentSize; + defaultEncryptedSegmentSizePresent = true; + } + if (!lastSegmentSizePresent) { + lastSegmentSize = segmentSize; + lastSegmentSizePresent = true; + } + if (!lastEncryptedSegmentSizePresent) { + lastEncryptedSegmentSize = encryptedSegmentSize; + lastEncryptedSegmentSizePresent = true; + } + } + + private static int trailingPadding(String hash) { + int padding = 0; + while (padding < hash.length() && hash.charAt(hash.length() - 1 - padding) == '=') { + padding++; + } + return padding; + } + + /** + * Every segment hash concatenated, base64-decoded if {@code decodeBase64} + * and taken as raw ASCII otherwise. This is the aggregate hash that the root + * signature and the assertion signatures are computed over. + */ + byte[] aggregate(boolean decodeBase64) { + if (size() == 0) { + return new byte[0]; + } + if (mutableSegments != null || !materializedSegments.isEmpty()) { + return aggregateMaterialized(decodeBase64); + } + byte[] scratch = new byte[stride]; + if (!decodeBase64) { + byte[] aggregate = new byte[Math.multiplyExact(count, stride)]; + for (int i = 0; i < count; i++) { + copyHash(i, aggregate, i * stride); + } + return aggregate; + } + + copyHash(0, scratch, 0); + int decodedStride = Base64.getDecoder().decode(scratch).length; + byte[] aggregate = new byte[Math.multiplyExact(count, decodedStride)]; + byte[] decoded = new byte[decodedStride]; + for (int i = 0; i < count; i++) { + copyHash(i, scratch, 0); + Base64.getDecoder().decode(scratch, decoded); + System.arraycopy(decoded, 0, aggregate, i * decodedStride, decodedStride); + } + return aggregate; + } + + private byte[] aggregateMaterialized(boolean decodeBase64) { + ByteArrayOutputStream aggregate = new ByteArrayOutputStream(); + for (int i = 0; i < size(); i++) { + String hash = getForRead(i).hash; + byte[] bytes = decodeBase64 + ? Base64.getDecoder().decode(hash) + : hash.getBytes(StandardCharsets.US_ASCII); + aggregate.write(bytes, 0, bytes.length); + } + return aggregate.toByteArray(); + } + + private void copyHash(int index, byte[] destination, int destinationOffset) { + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + System.arraycopy(chunk, (index % SEGMENTS_PER_CHUNK) * stride, destination, destinationOffset, stride); + } + + @Override + public Segment get(int index) { + if (mutableSegments != null) { + return mutableSegments.get(index); + } + Objects.checkIndex(index, count); + return materializedSegments.computeIfAbsent(index, this::unpack); + } + + Segment getForRead(int index) { + if (mutableSegments != null) { + return mutableSegments.get(index); + } + Objects.checkIndex(index, count); + Segment materialized = materializedSegments.get(index); + return materialized == null ? unpack(index) : materialized; + } + + private Segment unpack(int index) { + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + Segment segment = new Segment(); + segment.hash = new String(chunk, (index % SEGMENTS_PER_CHUNK) * stride, stride, StandardCharsets.US_ASCII); + boolean last = index == count - 1; + segment.segmentSize = last ? lastSegmentSize : defaultSegmentSize; + segment.encryptedSegmentSize = last ? lastEncryptedSegmentSize : defaultEncryptedSegmentSize; + segment.segmentSizePresent = last ? lastSegmentSizePresent : defaultSegmentSizePresent; + segment.encryptedSegmentSizePresent = last + ? lastEncryptedSegmentSizePresent : defaultEncryptedSegmentSizePresent; + return segment; + } + + private void materialize() { + if (mutableSegments != null) { + return; + } + List copy = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + copy.add(getForRead(i)); + } + mutableSegments = copy; + materializedSegments.clear(); + chunks.clear(); + } + + @Override + public Segment set(int index, Segment element) { + materialize(); + return mutableSegments.set(index, element); + } + + @Override + public void add(int index, Segment element) { + materialize(); + mutableSegments.add(index, element); + count = mutableSegments.size(); + modCount++; + } + + @Override + public Segment remove(int index) { + materialize(); + Segment removed = mutableSegments.remove(index); + count = mutableSegments.size(); + modCount++; + return removed; + } + + @Override + public int size() { + return mutableSegments == null ? count : mutableSegments.size(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof List) || size() != ((List) other).size()) { + return false; + } + List otherList = (List) other; + if (!(otherList instanceof Segments)) { + Iterator otherSegments = otherList.iterator(); + for (int i = 0; i < size(); i++) { + if (!Objects.equals(getForRead(i), otherSegments.next())) { + return false; + } + } + return true; + } + for (int i = 0; i < size(); i++) { + if (!Objects.equals(getForRead(i), ((Segments) otherList).getForRead(i))) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + for (int i = 0; i < size(); i++) { + Segment segment = getForRead(i); + hashCode = 31 * hashCode + (segment == null ? 0 : segment.hashCode()); + } + return hashCode; + } + } + + static Segment segmentAt(List segments, int index) { + return segments instanceof Segments + ? ((Segments) segments).getForRead(index) + : segments.get(index); + } + + /** + * Reads {@code segments} into a {@link Segments} when the manifest allows it, + * and into a plain {@code ArrayList} when it does not. Writing delegates to the + * reflective {@link Segment} adapter, so the JSON is unchanged either way. + */ + static final class SegmentsAdapterFactory implements TypeAdapterFactory { + @Override + @SuppressWarnings("unchecked") + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter element = gson.getAdapter(Segment.class); + TypeAdapter jsonElement = gson.getAdapter(JsonElement.class); + return (TypeAdapter) new SegmentsTypeAdapter(element, jsonElement); + } + + private static boolean hasValue(JsonObject object, String memberName) { + JsonElement member = object.get(memberName); + return member != null && !member.isJsonNull(); + } + } + + private static final class SegmentsTypeAdapter extends TypeAdapter> { + private final TypeAdapter elementAdapter; + private final TypeAdapter jsonElementAdapter; + + private SegmentsTypeAdapter(TypeAdapter elementAdapter, + TypeAdapter jsonElementAdapter) { + this.elementAdapter = elementAdapter; + this.jsonElementAdapter = jsonElementAdapter; + } + + @Override + public void write(JsonWriter out, List value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + out.beginArray(); + for (int i = 0; i < value.size(); i++) { + elementAdapter.write(out, segmentAt(value, i)); + } + out.endArray(); + } + + @Override + @SuppressWarnings("java:S1168") // Preserve JSON null so manifest validation rejects it. + public List read(JsonReader in) throws IOException { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return null; + } + in.beginArray(); + Segments compact = new Segments(); + List fallback = null; + while (in.hasNext()) { + Segment segment = readSegment(in); + if (fallback != null) { + fallback.add(segment); + } else if (!compact.append(segment)) { + fallback = new ArrayList<>(compact); + fallback.add(segment); + } + } + in.endArray(); + return fallback == null ? compact : fallback; + } + + private Segment readSegment(JsonReader in) throws IOException { + JsonElement rawSegment = jsonElementAdapter.read(in); + Segment segment = elementAdapter.fromJsonTree(rawSegment); + if (segment != null && rawSegment.isJsonObject()) { + JsonObject object = rawSegment.getAsJsonObject(); + segment.segmentSizePresent = SegmentsAdapterFactory.hasValue(object, "segmentSize"); + segment.encryptedSegmentSizePresent = SegmentsAdapterFactory.hasValue( + object, "encryptedSegmentSize"); + } + return segment; + } + } + static public class RootSignature { @SerializedName(value = "alg") public String algorithm; @@ -174,6 +596,7 @@ static public class IntegrityInformation { public String segmentHashAlg; public int segmentSizeDefault; public int encryptedSegmentSizeDefault; + @JsonAdapter(SegmentsAdapterFactory.class) public List segments; @Override @@ -220,17 +643,12 @@ public int hashCode() { * the two defaults against each other. */ private static class IntegrityInformationAdapterFactory implements TypeAdapterFactory { - private static final String SEGMENTS = "segments"; - private static final String SEGMENT_SIZE = "segmentSize"; - private static final String ENCRYPTED_SEGMENT_SIZE = "encryptedSegmentSize"; - @Override public TypeAdapter create(Gson gson, TypeToken type) { if (!IntegrityInformation.class.equals(type.getRawType())) { return null; } final TypeAdapter delegate = gson.getDelegateAdapter(this, type); - final TypeAdapter elementAdapter = gson.getAdapter(JsonElement.class); return new TypeAdapter() { @Override public void write(JsonWriter out, T value) throws IOException { @@ -239,30 +657,25 @@ public void write(JsonWriter out, T value) throws IOException { @Override public T read(JsonReader in) throws IOException { - JsonElement tree = elementAdapter.read(in); - T value = delegate.fromJsonTree(tree); - if (value instanceof IntegrityInformation && tree != null && tree.isJsonObject()) { - applySegmentSizeDefaults((IntegrityInformation) value, tree.getAsJsonObject()); + T value = delegate.read(in); + if (value instanceof IntegrityInformation) { + applySegmentSizeDefaults((IntegrityInformation) value); } return value; } }; } - private static void applySegmentSizeDefaults(IntegrityInformation integrityInformation, JsonObject json) { + private static void applySegmentSizeDefaults(IntegrityInformation integrityInformation) { List segments = integrityInformation.segments; - JsonElement rawSegments = json.get(SEGMENTS); - if (segments == null || rawSegments == null || rawSegments.isJsonNull()) { + if (segments == null) { // no segments to default; readManifest rejects this with its own message return; } - JsonArray rawSegmentArray = rawSegments.getAsJsonArray(); - if (segments.size() != rawSegmentArray.size()) { - // Gson's collection adapter is 1:1 with the JSON array, so this cannot happen - // today. asserting it rather than iterating the shorter of the two keeps a future - // filtering adapter from silently leaving the tail segments at 0 - throw new IllegalStateException("internal error: deserialized " + segments.size() - + " segments from a JSON array of " + rawSegmentArray.size()); + if (segments instanceof Segments) { + ((Segments) segments).applyDefaults(integrityInformation.segmentSizeDefault, + integrityInformation.encryptedSegmentSizeDefault); + return; } for (int i = 0; i < segments.size(); i++) { Segment segment = segments.get(i); @@ -270,21 +683,14 @@ private static void applySegmentSizeDefaults(IntegrityInformation integrityInfor // a null entry in the array; readManifest rejects it with its own message continue; } - JsonObject rawSegment = rawSegmentArray.get(i).getAsJsonObject(); - if (!hasValue(rawSegment, SEGMENT_SIZE)) { + if (!segment.segmentSizePresent) { segment.segmentSize = integrityInformation.segmentSizeDefault; } - if (!hasValue(rawSegment, ENCRYPTED_SEGMENT_SIZE)) { + if (!segment.encryptedSegmentSizePresent) { segment.encryptedSegmentSize = integrityInformation.encryptedSegmentSizeDefault; } } } - - /** Whether {@code memberName} carries a value; an explicit {@code null} counts as absent. */ - private static boolean hasValue(JsonObject object, String memberName) { - JsonElement member = object.get(memberName); - return member != null && !member.isJsonNull(); - } } /** @@ -316,16 +722,12 @@ private static boolean hasValue(JsonObject object, String memberName) { * document instead of preferring {@code schemaVersion}. */ private static class SpecVersionAdapterFactory implements TypeAdapterFactory { - private static final String NON_ALIGNED_SPEC_VERSION = "tdf_spec_version"; - private static final String PAYLOAD = "payload"; - @Override public TypeAdapter create(Gson gson, TypeToken type) { if (!Manifest.class.equals(type.getRawType())) { return null; } final TypeAdapter delegate = gson.getDelegateAdapter(this, type); - final TypeAdapter elementAdapter = gson.getAdapter(JsonElement.class); return new TypeAdapter() { @Override public void write(JsonWriter out, T value) throws IOException { @@ -334,12 +736,11 @@ public void write(JsonWriter out, T value) throws IOException { @Override public T read(JsonReader in) throws IOException { - JsonElement tree = elementAdapter.read(in); - T value = delegate.fromJsonTree(tree); - if (value instanceof Manifest && tree != null && tree.isJsonObject()) { + T value = delegate.read(in); + if (value instanceof Manifest) { Manifest manifest = (Manifest) value; if (manifest.tdfVersion == null || manifest.tdfVersion.isEmpty()) { - String nonAligned = nonAlignedSpecVersion(tree.getAsJsonObject()); + String nonAligned = nonAlignedSpecVersion(manifest); if (nonAligned != null) { manifest.tdfVersion = nonAligned; } @@ -351,16 +752,12 @@ public T read(JsonReader in) throws IOException { } /** The first non-empty string under the non-aligned name, root before payload, or null. */ - private static String nonAlignedSpecVersion(JsonObject root) { - String atRoot = nonEmptyString(root.get(NON_ALIGNED_SPEC_VERSION)); + private static String nonAlignedSpecVersion(Manifest manifest) { + String atRoot = nonEmptyString(manifest.nonAlignedSpecVersion); if (atRoot != null) { return atRoot; } - JsonElement payload = root.get(PAYLOAD); - if (payload != null && payload.isJsonObject()) { - return nonEmptyString(payload.getAsJsonObject().get(NON_ALIGNED_SPEC_VERSION)); - } - return null; + return manifest.payload == null ? null : nonEmptyString(manifest.payload.nonAlignedSpecVersion); } private static String nonEmptyString(JsonElement element) { @@ -484,6 +881,9 @@ static public class Payload { public String protocol; public String mimeType; public boolean isEncrypted; + @DeserializeOnly + @SerializedName("tdf_spec_version") + JsonElement nonAlignedSpecVersion; @Override public boolean equals(Object o) { @@ -764,47 +1164,107 @@ public AssertionConfig.Statement deserialize(JsonElement json, Type typeOfT, Jso public Payload payload; public List assertions = new ArrayList<>(); protected static Manifest readManifest(String manifestJson) { - Manifest result = gson.fromJson(manifestJson, Manifest.class); + return readManifest(new StringReader(manifestJson)); + } + + /** + * Parses a manifest without materializing it as a {@link String}, which a + * manifest with tens of millions of segments cannot be. + */ + protected static Manifest readManifest(Reader manifestJson) { + FailureTrackingReader reader = new FailureTrackingReader(manifestJson); + Manifest result; + try { + result = gson.fromJson(reader, Manifest.class); + } catch (JsonParseException e) { + if (reader.failure != null) { + throw new SDKException("error reading manifest from zip file", reader.failure); + } + throw e; + } if (result == null) { throw new IllegalArgumentException("Manifest is empty"); } + validateManifest(result); + return result; + } + + private static final class FailureTrackingReader extends Reader { + private final Reader delegate; + private IOException failure; + + private FailureTrackingReader(Reader delegate) { + this.delegate = delegate; + } + + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + try { + return delegate.read(buffer, offset, length); + } catch (IOException e) { + failure = e; + throw e; + } + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } + + private static void validateManifest(Manifest result) { if (result.assertions == null) { result.assertions = new ArrayList<>(); } - if (result.payload == null) { - throw new IllegalArgumentException("Manifest with null payload"); - } else if (result.encryptionInformation == null) { - throw new IllegalArgumentException("Manifest with null encryptionInformation"); - } else if (result.encryptionInformation.integrityInformation == null) { - throw new IllegalArgumentException("Manifest with null integrityInformation"); - } else if (result.encryptionInformation.integrityInformation.rootSignature == null) { - throw new IllegalArgumentException("Manifest with null rootSignature"); - } else if (result.encryptionInformation.integrityInformation.rootSignature.algorithm == null - || result.encryptionInformation.integrityInformation.rootSignature.signature == null) { + requireManifestField(result.payload, "Manifest with null payload"); + requireManifestField(result.encryptionInformation, "Manifest with null encryptionInformation"); + + var encryptionInformation = result.encryptionInformation; + requireManifestField(encryptionInformation.integrityInformation, "Manifest with null integrityInformation"); + requireManifestField(encryptionInformation.keyAccessObj, "Manifest with null keyAccessObj"); + requireManifestField(encryptionInformation.policy, "Manifest with null policy"); + + var integrityInformation = encryptionInformation.integrityInformation; + requireManifestField(integrityInformation.rootSignature, "Manifest with null rootSignature"); + requireManifestField(integrityInformation.segments, "Manifest with null segments"); + + var rootSignature = integrityInformation.rootSignature; + if (rootSignature.algorithm == null || rootSignature.signature == null) { throw new IllegalArgumentException("Manifest with invalid rootSignature"); - } else if (result.encryptionInformation.integrityInformation.segments == null) { - throw new IllegalArgumentException("Manifest with null segments"); - } else if (result.encryptionInformation.keyAccessObj == null) { - throw new IllegalArgumentException("Manifest with null keyAccessObj"); - } else if (result.encryptionInformation.policy == null) { - throw new IllegalArgumentException("Manifest with null policy"); } - for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { - if (segment == null || segment.hash == null) { - throw new IllegalArgumentException("Invalid integrity segment"); + validateSegments(integrityInformation.segments); + validateKeyAccess(encryptionInformation.keyAccessObj); + } + + private static void requireManifestField(Object field, String message) { + if (field == null) { + throw new IllegalArgumentException(message); + } + } + + private static void validateSegments(List segments) { + // Segments rejects null segments and hashes as it is built, so only the + // fallback representation needs checking here. + if (!(segments instanceof Segments)) { + for (Manifest.Segment segment : segments) { + if (segment == null || segment.hash == null) { + throw new IllegalArgumentException("Invalid integrity segment"); + } } } - for (Manifest.KeyAccess keyAccess : result.encryptionInformation.keyAccessObj) { + } + + private static void validateKeyAccess(List keyAccessObjects) { + for (Manifest.KeyAccess keyAccess : keyAccessObjects) { if (keyAccess == null) { throw new IllegalArgumentException("Invalid null KeyAccess in manifest"); } else if (keyAccess.url == null) { throw new IllegalArgumentException("Invalid KeyAccess with null url in manifest"); } } - - return result; } static PolicyObject decodePolicyObject(Manifest manifest) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index f69b6896..f4aa7d63 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -196,8 +196,9 @@ public static boolean isTDF(SeekableByteChannel channel) { */ public static Manifest readManifest(SeekableByteChannel tdfBytes) throws SDKException, IOException { TDFReader reader = new TDFReader(tdfBytes); - String manifestJson = reader.manifest(); - return Manifest.readManifest(manifestJson); + try (var manifestJson = reader.manifest()) { + return Manifest.readManifest(manifestJson); + } } /** diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index fddead94..883d885d 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -15,10 +15,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.security.*; @@ -81,6 +83,13 @@ private static byte[] tdfECKeySaltCompute() { private static final Gson gson = new GsonBuilder().create(); + /** + * The manifest is serialized directly into the zip entry, and + * {@link ZipWriter#stream} updates its CRC on every {@code write} call, so the + * character encoder needs a buffer in front of it. + */ + private static final int MANIFEST_WRITE_BUFFER_SIZE = 1 << 16; + /** * An AES-GCM IV generator using the deterministic construction of NIST SP 800-38D * section 8.2.1: a 64-bit fixed field followed by a 32-bit big-endian invocation field. @@ -398,7 +407,9 @@ public void readPayload(OutputStream outputStream) throws SDK.TamperException, I validateSegmentSizes(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { + List segments = manifest.encryptionInformation.integrityInformation.segments; + for (int i = 0; i < segments.size(); i++) { + Manifest.Segment segment = Manifest.segmentAt(segments, i); byte[] readBuf = new byte[(int) segment.encryptedSegmentSize]; int bytesRead = tdfReader.readPayloadBytes(readBuf); @@ -413,7 +424,7 @@ public void readPayload(OutputStream outputStream) throws SDK.TamperException, I var sigAlg = segmentIntegrityAlgorithmFromManifest( manifest.encryptionInformation.integrityInformation.segmentHashAlg); - var payloadSig = segmentIntegrity(readBuf, payloadKey, sigAlg); + var payloadSig = calculateSignature(readBuf, payloadKey, sigAlg); if (isLegacyTdf) { payloadSig = Hex.encodeHexString(payloadSig).getBytes(StandardCharsets.UTF_8); } @@ -453,7 +464,7 @@ private void validateSegmentSizes() { List segments = manifest.encryptionInformation.integrityInformation.segments; for (int i = 0; i < segments.size(); i++) { - long encryptedSegmentSize = segments.get(i).encryptedSegmentSize; + long encryptedSegmentSize = Manifest.segmentAt(segments, i).encryptedSegmentSize; if (encryptedSegmentSize < minEncryptedSegmentSize) { throw new IllegalStateException("invalid TDF: segment " + i @@ -499,66 +510,33 @@ private static Config.IntegrityAlgorithm segmentIntegrityAlgorithmFromManifest(S } /** - * Recovers the trailing AES-GCM authentication tag from a segment's ciphertext. - *

- * Recovering a tag is not verifying one. These are bytes whoever supplied the input - * already holds, so comparing them against a manifest value is keyless and on its own - * proves nothing — an attacker can re-chunk a payload and write each chunk's own - * trailing sixteen bytes into its {@code segment.hash}. What makes a GMAC segment hash - * trustworthy is the keyed checks around it: {@code loadTDF} has already validated the - * whole list of segment hashes against the HS256 root signature, and {@code readPayload} - * follows the comparison with a real AES-GCM tag check under the payload key. - *

- * The root signature has neither backstop — it is the outermost check, so a "GMAC root" - * is a keyless comparison with nothing behind it. The asymmetry is therefore structural, - * not a property of the bytes, and it is why {@link #rootIntegrity} does not offer this - * algorithm. + * Concatenates every segment hash, which is what the root signature and the + * assertion signatures are computed over. */ - private static byte[] aeadTag(byte[] ciphertext) { - if (kGMACPayloadLength > ciphertext.length) { - throw new IllegalArgumentException("tried to calculate GMAC on too small a payload. payload is " - + ciphertext.length + " bytes while GMAC is " + kGMACPayloadLength + " bytes"); + private static byte[] aggregateSegmentHashes(List segments, boolean isEncrypted) { + if (segments instanceof Manifest.Segments) { + return ((Manifest.Segments) segments).aggregate(isEncrypted); } - - return Arrays.copyOfRange(ciphertext, ciphertext.length - kGMACPayloadLength, ciphertext.length); + var aggregateHash = new ByteArrayOutputStream(); + for (Manifest.Segment segment : segments) { + aggregateHash.writeBytes(isEncrypted + ? Base64.getDecoder().decode(segment.hash) + : segment.hash.getBytes()); + } + return aggregateHash.toByteArray(); } - /** - * The integrity value recorded in a segment's {@code hash}. - * - * @param ciphertext the AES-GCM output for this segment, whole and unmodified - * @param key the payload key - * @param algorithm {@code GMAC} to reuse the segment's own AEAD tag, or - * {@code HS256} to HMAC the segment ciphertext - * @throws IllegalArgumentException if {@code algorithm} is null or unsupported - */ - static byte[] segmentIntegrity(byte[] ciphertext, byte[] key, Config.IntegrityAlgorithm algorithm) { - requireSupportedSegmentIntegrityAlgorithm(algorithm); - switch (algorithm) { - case HS256: - return CryptoUtils.CalculateSHA256Hmac(key, ciphertext); - case GMAC: - return aeadTag(ciphertext); - default: - throw new IllegalArgumentException("unsupported segment integrity algorithm: " + algorithm); + private static byte[] calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) { + if (algorithm == Config.IntegrityAlgorithm.HS256) { + return CryptoUtils.CalculateSHA256Hmac(secret, data); } - } - /** - * The integrity value recorded in {@code rootSignature.sig}, over the concatenated - * segment hashes. - *

- * HS256 only. The aggregate hash never passes through the AEAD, so there is no tag - * to recover from it; a "GMAC" root signature is just a copy of the last segment - * hash, which is attacker-controlled manifest data. Accepting one would let anyone - * truncate, reorder, duplicate or drop segments without holding a key, since nothing - * else binds a segment to its index or to the segment count. - * - * @throws IllegalArgumentException if {@code algorithm} is anything but HS256 - */ - static byte[] rootIntegrity(byte[] aggregateHash, byte[] key, Config.IntegrityAlgorithm algorithm) { - requireSupportedRootIntegrityAlgorithm(algorithm); - return CryptoUtils.CalculateSHA256Hmac(key, aggregateHash); + if (kGMACPayloadLength > data.length) { + throw new IllegalArgumentException("tried to calculate GMAC on too small a payload. payload is " + + data.length + " bytes while GMAC is " + kGMACPayloadLength + " bytes"); + } + + return Arrays.copyOfRange(data, data.length - kGMACPayloadLength, data.length); } /** @@ -650,10 +628,10 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo long encryptedSegmentSize = (long) tdfConfig.defaultSegmentSize + kGcmIvSize + AesGcm.GCM_TAG_LENGTH; TDFWriter tdfWriter = new TDFWriter(outputStream); - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); byte[] readBuf = new byte[tdfConfig.defaultSegmentSize]; - tdfObject.manifest.encryptionInformation.integrityInformation.segments = new ArrayList<>(); + var segments = new Manifest.Segments(); + tdfObject.manifest.encryptionInformation.integrityInformation.segments = segments; boolean finished; try (var payloadOutput = tdfWriter.payload()) { do { @@ -667,30 +645,33 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo byte[] cipherData; byte[] segmentSig; - Manifest.Segment segmentInfo = new Manifest.Segment(); // encrypt cipherData = tdfObject.aesGcm.encrypt(streamIv.next(), AesGcm.GCM_TAG_LENGTH, readBuf, 0, readThisLoop); payloadOutput.write(cipherData); - segmentSig = segmentIntegrity(cipherData, tdfObject.payloadKey, tdfConfig.segmentIntegrityAlgorithm); + segmentSig = calculateSignature(cipherData, tdfObject.payloadKey, tdfConfig.segmentIntegrityAlgorithm); if (tdfConfig.hexEncodeRootAndSegmentHashes) { segmentSig = Hex.encodeHexString(segmentSig).getBytes(StandardCharsets.UTF_8); } - segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig); - - aggregateHash.write(segmentSig); - segmentInfo.segmentSize = readThisLoop; - segmentInfo.encryptedSegmentSize = cipherData.length; - tdfObject.manifest.encryptionInformation.integrityInformation.segments.add(segmentInfo); + // Every segment we write is full except the last one, and every hash has the + // same length, so the compact representation always accepts them. + if (!segments.append(Base64.getEncoder().encodeToString(segmentSig), readThisLoop, + cipherData.length)) { + throw new SDKException("unable to record segment " + segments.size() + " in the manifest"); + } } while (!finished); } + // Materialized once and reused by the root signature and every assertion below; + // it is proportional to the number of segments. + byte[] aggregateHash = segments.aggregate(true); + Manifest.RootSignature rootSignature = new Manifest.RootSignature(); - byte[] rootSig = rootIntegrity(aggregateHash.toByteArray(), tdfObject.payloadKey, + byte[] rootSig = calculateSignature(aggregateHash, tdfObject.payloadKey, tdfConfig.integrityAlgorithm); byte[] encodedRootSig = tdfConfig.hexEncodeRootAndSegmentHashes ? Hex.encodeHexString(rootSig).getBytes(StandardCharsets.UTF_8) @@ -742,9 +723,9 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo throw new SDKException("error decoding assertion hash", e); } } - byte[] completeHash = new byte[aggregateHash.size() + assertionHash.length]; - System.arraycopy(aggregateHash.toByteArray(), 0, completeHash, 0, aggregateHash.size()); - System.arraycopy(assertionHash, 0, completeHash, aggregateHash.size(), assertionHash.length); + byte[] completeHash = new byte[aggregateHash.length + assertionHash.length]; + System.arraycopy(aggregateHash, 0, completeHash, 0, aggregateHash.length); + System.arraycopy(assertionHash, 0, completeHash, aggregateHash.length, assertionHash.length); var encodedHash = Base64.getEncoder().encodeToString(completeHash); @@ -765,9 +746,13 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo } tdfObject.manifest.assertions = signedAssertions; - String manifestAsStr = gson.toJson(tdfObject.manifest); - tdfWriter.appendManifest(manifestAsStr); + // Serialize straight into the zip entry: a manifest with tens of millions of segments + // exceeds the maximum size of a Java String. + try (var manifestOutput = new BufferedWriter( + new OutputStreamWriter(tdfWriter.manifest(), StandardCharsets.UTF_8), MANIFEST_WRITE_BUFFER_SIZE)) { + gson.toJson(tdfObject.manifest, manifestOutput); + } tdfObject.size = tdfWriter.finish(); return tdfObject; @@ -804,9 +789,11 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig, Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throws SDKException, IOException { TDFReader tdfReader = new TDFReader(tdf); - String manifestJson = tdfReader.manifest(); // use Manifest.readManifest in order to validate the Manifest input - Manifest manifest = Manifest.readManifest(manifestJson); + Manifest manifest; + try (var manifestJson = tdfReader.manifest()) { + manifest = Manifest.readManifest(manifestJson); + } byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; @@ -891,22 +878,15 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) String rootAlgorithm = manifest.encryptionInformation.integrityInformation.rootSignature.algorithm; String rootSignature = manifest.encryptionInformation.integrityInformation.rootSignature.signature; - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - if (manifest.payload.isEncrypted) { - byte[] decodedHash = Base64.getDecoder().decode(segment.hash); - aggregateHash.write(decodedHash); - } else { - aggregateHash.write(segment.hash.getBytes()); - } - } + byte[] aggregateHash = aggregateSegmentHashes( + manifest.encryptionInformation.integrityInformation.segments, manifest.payload.isEncrypted); String rootSigValue; boolean isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); if (manifest.payload.isEncrypted) { var sigAlg = rootIntegrityAlgorithmFromManifest(rootAlgorithm); - var sig = rootIntegrity(aggregateHash.toByteArray(), payloadKey, sigAlg); + var sig = calculateSignature(aggregateHash, payloadKey, sigAlg); if (isLegacyTdf) { sig = Hex.encodeHexString(sig).getBytes(); } @@ -926,7 +906,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new IllegalStateException("error getting instance of SHA-256 digest", e); } - rootSigValue = Base64.getEncoder().encodeToString(digest.digest(aggregateHash.toString().getBytes())); + // the round trip through the platform default charset is a no-op for the + // hex hashes this branch sees, and is kept so the digest is unchanged + rootSigValue = Base64.getEncoder().encodeToString(digest.digest(new String(aggregateHash).getBytes())); } if (rootSignature.compareTo(rootSigValue) != 0) { @@ -941,7 +923,6 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) "segment size mismatch. encrypted segment size differs from plaintext segment size. the TDF is invalid"); } - var aggregateHashByteArrayBytes = aggregateHash.toByteArray(); // Validate assertions for (var assertion : manifest.assertions) { // Skip assertion verification if disabled @@ -981,9 +962,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new SDKException("error decoding assertion hash", e); } } - var signature = new byte[aggregateHashByteArrayBytes.length + hashOfAssertion.length]; - System.arraycopy(aggregateHashByteArrayBytes, 0, signature, 0, aggregateHashByteArrayBytes.length); - System.arraycopy(hashOfAssertion, 0, signature, aggregateHashByteArrayBytes.length, hashOfAssertion.length); + var signature = new byte[aggregateHash.length + hashOfAssertion.length]; + System.arraycopy(aggregateHash, 0, signature, 0, aggregateHash.length); + System.arraycopy(hashOfAssertion, 0, signature, aggregateHash.length, hashOfAssertion.length); var encodeSignature = Base64.getEncoder().encodeToString(signature); if (!Objects.equals(encodeSignature, hashValues.getSignature())) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 93548d67..43fadb2a 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -1,8 +1,10 @@ package io.opentdf.platform.sdk; -import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -19,6 +21,8 @@ */ public class TDFReader { + private static final int MANIFEST_BUFFER_SIZE = 1 << 16; + private final ZipReader.Entry manifestEntry; private final InputStream payload; @@ -49,15 +53,18 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); } - String manifest() { - var out = new ByteArrayOutputStream(); + /** + * The manifest entry as a character stream; the caller must close it. Returned + * as a stream rather than a String because a manifest with tens of millions of + * segments exceeds the maximum size of a Java String. + */ + Reader manifest() { try { - manifestEntry.getData().transferTo(out); + return new BufferedReader( + new InputStreamReader(manifestEntry.getData(), StandardCharsets.UTF_8), MANIFEST_BUFFER_SIZE); } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } - - return out.toString(StandardCharsets.UTF_8); } int readPayloadBytes(byte[] buf) { @@ -74,8 +81,10 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - String manifestJson = manifest(); - Manifest manifest = Manifest.readManifest(manifestJson); - return Manifest.decodePolicyObject(manifest); + try (Reader manifestRaw = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestRaw)); + } catch (IOException e) { + throw new SDKException("error reading manifest from zip file", e); + } } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 93046f83..6b6a8b27 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -41,8 +41,25 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } + /** + * @deprecated use {@link #manifest()}. A manifest with tens of millions of segments + * exceeds the maximum size of a Java {@link String}, so this cannot + * express every manifest the SDK writes. It produces the same zip entry. + */ + @SuppressWarnings("java:S1133") // Retained for source and binary compatibility. + @Deprecated(since = "0.20.0", forRemoval = false) public void appendManifest(String manifest) throws IOException { - this.archiveWriter.data(TDF_MANIFEST_FILE_NAME, manifest.getBytes(StandardCharsets.UTF_8)); + try (OutputStream output = manifest()) { + output.write(manifest.getBytes(StandardCharsets.UTF_8)); + } + } + + /** + * Opens the manifest entry for writing. The returned stream must be closed before + * {@link #finish()} is called, otherwise the entry never makes it into the central directory. + */ + public OutputStream manifest() throws IOException { + return this.archiveWriter.stream(TDF_MANIFEST_FILE_NAME); } public OutputStream payload() throws IOException { @@ -53,4 +70,4 @@ public OutputStream payload() throws IOException { public long finish() throws IOException { return this.archiveWriter.finish(); } -} \ No newline at end of file +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java index 04070a61..c49a709d 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/ManifestTest.java @@ -1,6 +1,7 @@ package io.opentdf.platform.sdk; import com.google.gson.Gson; +import com.google.gson.JsonParseException; import com.google.gson.JsonParser; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -8,6 +9,8 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.io.EOFException; +import java.io.Reader; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -271,6 +274,61 @@ void rejectsKeyAccessWithoutUrl() { .hasMessage("Invalid KeyAccess with null url in manifest"); } + @Test + void wrapsManifestReaderIoFailures() { + Reader failingReader = new Reader() { + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + throw new IOException("manifest read failed"); + } + + @Override + public void close() { + // No resources to release. + } + }; + + assertThatThrownBy(() -> Manifest.readManifest(failingReader)) + .isInstanceOf(SDKException.class) + .hasMessage("error reading manifest from zip file") + .hasRootCauseMessage("manifest read failed"); + } + + @Test + void preservesMalformedJsonErrors() { + assertThatThrownBy(() -> Manifest.readManifest("{\"payload\":")) + .isInstanceOf(JsonParseException.class) + .isNotInstanceOf(SDKException.class); + } + + @Test + void wrapsEofThrownByTheSourceReader() { + Reader failingReader = new Reader() { + private final String partialManifest = "{\"payload\":"; + private int position; + + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + if (position == partialManifest.length()) { + throw new EOFException("archive ended early"); + } + int copied = Math.min(length, partialManifest.length() - position); + partialManifest.getChars(position, position + copied, buffer, offset); + position += copied; + return copied; + } + + @Override + public void close() { + // No resources to release. + } + }; + + assertThatThrownBy(() -> Manifest.readManifest(failingReader)) + .isInstanceOf(SDKException.class) + .hasRootCauseMessage("archive ended early"); + } + @Test void testReadingManifestWithObjectStatementValue() throws IOException { final Manifest manifest; diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java new file mode 100644 index 00000000..9333d57c --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -0,0 +1,275 @@ +package io.opentdf.platform.sdk; + +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Covers the compact {@link Manifest.Segments} representation: it has to + * serialize exactly like a plain list, fall back when a manifest does not fit + * its assumptions, and let a manifest larger than a Java String round trip. + */ +public class SegmentsTest { + + private static final Gson GSON = new Gson(); + private static final int SEGMENT_SIZE = 16384; + private static final int ENCRYPTED_SEGMENT_SIZE = SEGMENT_SIZE + 28; + + private static String hash(int i) { + var bytes = new byte[16]; + bytes[0] = (byte) i; + bytes[1] = (byte) (i >> 8); + bytes[2] = (byte) (i >> 16); + return Base64.getEncoder().encodeToString(bytes); + } + + private static Manifest.Segment segment(String hash, long segmentSize, long encryptedSegmentSize) { + var segment = new Manifest.Segment(); + segment.hash = hash; + segment.segmentSize = segmentSize; + segment.encryptedSegmentSize = encryptedSegmentSize; + return segment; + } + + private static String manifestJson(List segments) { + return "{\"schemaVersion\":\"4.3.0\",\"encryptionInformation\":{\"type\":\"split\"," + // no '=' padding: Gson HTML-escapes it, and this literal is compared verbatim + + "\"policy\":\"eyJib2R5Ijp7fX0K\",\"keyAccess\":[{\"type\":\"wrapped\",\"url\":\"http://kas\"," + + "\"protocol\":\"kas\",\"wrappedKey\":\"AAAA\"}]," + + "\"method\":{\"algorithm\":\"AES-256-GCM\",\"iv\":\"AAAAAAAAAAAA\",\"isStreamable\":true}," + + "\"integrityInformation\":{\"rootSignature\":{\"alg\":\"GMAC\",\"sig\":\"AAAA\"}," + + "\"segmentHashAlg\":\"GMAC\",\"segmentSizeDefault\":" + SEGMENT_SIZE + + ",\"encryptedSegmentSizeDefault\":" + ENCRYPTED_SEGMENT_SIZE + ",\"segments\":" + + segments.stream().map(GSON::toJson).collect(Collectors.joining(",", "[", "]")) + + "}},\"payload\":{\"type\":\"reference\",\"url\":\"0.payload\",\"protocol\":\"zip\"," + + "\"mimeType\":\"application/octet-stream\",\"isEncrypted\":true},\"assertions\":[]}"; + } + + /** A run of uniform segments with a short final one, i.e. what this SDK writes. */ + private static List uniformSegments(int count) { + return IntStream.range(0, count) + .mapToObj(i -> segment(hash(i), + i == count - 1 ? 7 : SEGMENT_SIZE, + i == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)) + .collect(Collectors.toList()); + } + + @Test + void uniformSegmentsAreStoredCompactlyAndSerializeUnchanged() { + var segments = uniformSegments(5); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed) + .isInstanceOf(Manifest.Segments.class) + .containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void irregularSegmentSizesFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + // a non-final segment that disagrees with the defaults cannot be stored compactly + segments.set(1, segment(hash(1), SEGMENT_SIZE / 2, ENCRYPTED_SEGMENT_SIZE / 2)); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed) + .isNotInstanceOf(Manifest.Segments.class) + .containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void segmentsWithDifferingHashLengthsFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + segments.set(1, segment("deadbeef", SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + var json = manifestJson(segments); + + var parsed = Manifest.readManifest(json).encryptionInformation.integrityInformation.segments; + + assertThat(parsed) + .isNotInstanceOf(Manifest.Segments.class) + .containsExactlyElementsOf(segments); + } + + @Test + void anAbsurdlyLongHashFallsBackInsteadOfSizingAChunkFromIt() { + // a chunk is stride * 4096 bytes, so without a bound this one hash would ask for + // gigabytes before the fallback ever ran + var segments = new ArrayList<>(uniformSegments(2)); + segments.set(0, segment("A".repeat(100_000), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + + var parsed = Manifest.readManifest(manifestJson(segments)) + .encryptionInformation.integrityInformation.segments; + + assertThat(parsed) + .isNotInstanceOf(Manifest.Segments.class) + .containsExactlyElementsOf(segments); + } + + @Test + void aggregateConcatenatesEveryHash() { + var segments = new Manifest.Segments(); + for (int i = 0; i < 3; i++) { + assertThat(segments.append(hash(i), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)).isTrue(); + } + + var decoded = new byte[48]; + for (int i = 0; i < 3; i++) { + System.arraycopy(Base64.getDecoder().decode(hash(i)), 0, decoded, i * 16, 16); + } + assertThat(segments.aggregate(true)).isEqualTo(decoded); + assertThat(segments.aggregate(false)) + .isEqualTo((hash(0) + hash(1) + hash(2)).getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + @Test + void parsedCompactSegmentsPreserveElementAndListMutations() { + var manifest = Manifest.readManifest(manifestJson(uniformSegments(3))); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + assertThat(parsed).isInstanceOf(Manifest.Segments.class); + + Manifest.Segment first = parsed.get(0); + first.hash = hash(10); + first.segmentSize = 123; + + Manifest.Segment replacement = segment(hash(11), 456, 484); + Manifest.Segment replaced = parsed.set(1, replacement); + Manifest.Segment added = segment(hash(12), 789, 817); + parsed.add(added); + Manifest.Segment removed = parsed.remove(2); + + assertThat(replaced.hash).isEqualTo(hash(1)); + assertThat(removed.hash).isEqualTo(hash(2)); + assertThat(parsed).containsExactly(first, replacement, added); + + var roundTripped = Manifest.readManifest(Manifest.toJson(manifest)) + .encryptionInformation.integrityInformation.segments; + assertThat(roundTripped).containsExactlyElementsOf(parsed); + } + + @Test + void emptySegmentsAggregateToNothing() { + assertThat(new Manifest.Segments().aggregate(true)).isEmpty(); + assertThat(new Manifest.Segments().aggregate(false)).isEmpty(); + } + + @Test + void compactSegmentsRetainListEqualityAndHashing() { + var left = new Manifest.Segments(); + var right = new Manifest.Segments(); + var expected = uniformSegments(3); + for (Manifest.Segment segment : expected) { + assertThat(left.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)).isTrue(); + assertThat(right.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)).isTrue(); + } + + assertThat(left).isEqualTo(right); + assertThat(left).hasSameHashCodeAs(right); + assertThat(left).containsExactlyElementsOf(expected); + } + + /** + * A manifest whose JSON is larger than {@code Integer.MAX_VALUE} cannot be held + * as a String, so both directions have to stream. Opt in with + * {@code -Dtdf.hugeSegments=true -Xmx2g}; it needs roughly a minute and 1 GiB. + */ + @Test + @EnabledIfSystemProperty(named = "tdf.hugeSegments", matches = "true") + void aManifestTooLargeToBeAStringRoundTrips() throws IOException { + // ~25M segments at ~99 JSON bytes each is ~2.5 GB, past the String ceiling + final int count = 25_000_000; + + Manifest manifest; + try (Reader json = new GeneratedManifestReader(count)) { + manifest = Manifest.readManifest(json); + } + var segments = manifest.encryptionInformation.integrityInformation.segments; + assertThat(segments) + .isInstanceOf(Manifest.Segments.class) + .hasSize(count); + assertThat(segments.get(count - 1).hash).isEqualTo(hash(count - 1)); + + var counting = new CountingWriter(); + new Gson().toJson(manifest, counting); + assertThat(counting.written).isGreaterThan(Integer.MAX_VALUE); + } + + /** Renders a manifest with {@code count} segments without ever storing it. */ + private static final class GeneratedManifestReader extends Reader { + private final int count; + private final String suffix; + private int next; + private String pending; + private int pendingOffset; + + GeneratedManifestReader(int count) { + this.count = count; + var template = manifestJson(uniformSegments(1)); + this.pending = template.substring(0, template.indexOf("\"segments\":[") + "\"segments\":[".length()); + this.suffix = template.substring(template.indexOf("]}},\"payload\"")); + } + + @Override + public int read(char[] buffer, int offset, int length) { + if (pendingOffset == pending.length()) { + if (next > count) { + return -1; + } + pending = next == count ? suffix + : (next == 0 ? "" : ",") + GSON.toJson(segment(hash(next), + next == count - 1 ? 7 : SEGMENT_SIZE, + next == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)); + pendingOffset = 0; + next++; + if (pending.isEmpty()) { + return read(buffer, offset, length); + } + } + int n = Math.min(length, pending.length() - pendingOffset); + pending.getChars(pendingOffset, pendingOffset + n, buffer, offset); + pendingOffset += n; + return n; + } + + @Override + public void close() { + // No resources to release. + } + } + + private static final class CountingWriter extends Writer { + private long written; + + @Override + public void write(char[] buffer, int offset, int length) { + written += length; + } + + @Override + public void flush() { + // Nothing is buffered. + } + + @Override + public void close() { + // No resources to release. + } + } +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFReaderTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFReaderTest.java index 8844a002..43f2a02d 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFReaderTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFReaderTest.java @@ -7,6 +7,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.Reader; +import java.io.StringWriter; import java.nio.charset.StandardCharsets; import static org.assertj.core.api.Assertions.assertThat; @@ -64,12 +66,20 @@ private static SeekableInMemoryByteChannel archiveOf(String... namesAndContents) return new SeekableInMemoryByteChannel(out.toByteArray()); } + private static String manifest(TDFReader reader) throws IOException { + try (Reader manifest = reader.manifest()) { + var text = new StringWriter(); + manifest.transferTo(text); + return text.toString(); + } + } + @Test void readsManifestUnderTheSpecName() throws IOException { try (var tdf = archiveOf( TDFWriter.TDF_PAYLOAD_FILE_NAME, PAYLOAD, "manifest.json", MANIFEST)) { - assertThat(new TDFReader(tdf).manifest()).isEqualTo(MANIFEST); + assertThat(manifest(new TDFReader(tdf))).isEqualTo(MANIFEST); } } @@ -78,7 +88,7 @@ void readsManifestUnderTheOffspecName() throws IOException { try (var tdf = archiveOf( TDFWriter.TDF_PAYLOAD_FILE_NAME, PAYLOAD, "0.manifest.json", MANIFEST)) { - assertThat(new TDFReader(tdf).manifest()).isEqualTo(MANIFEST); + assertThat(manifest(new TDFReader(tdf))).isEqualTo(MANIFEST); } } @@ -107,7 +117,7 @@ void prefersTheSpecNameWhenAnArchiveCarriesBoth() throws IOException { TDFWriter.TDF_PAYLOAD_FILE_NAME, PAYLOAD, "0.manifest.json", OTHER_MANIFEST, "manifest.json", MANIFEST)) { - assertThat(new TDFReader(tdf).manifest()).isEqualTo(MANIFEST); + assertThat(manifest(new TDFReader(tdf))).isEqualTo(MANIFEST); } } diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFRootSignatureTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFRootSignatureTest.java index f8ccc0c3..1d1ffa0a 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFRootSignatureTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFRootSignatureTest.java @@ -25,6 +25,7 @@ import java.util.function.UnaryOperator; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** @@ -435,35 +436,35 @@ void createTdfRefusesAnUnsetSegmentAlgorithm() { /* * The manifest resolvers above are what a hostile file actually meets, and the * round-trip tests cover them. These call the inner guards directly because nothing - * else does: with the resolvers in place a GMAC root cannot reach `rootIntegrity`, - * so without these a regression that reintroduced tag extraction there would leave - * the whole suite green. Defence in depth is only depth if the inner layer is held - * to its contract independently. + * else does: with the resolvers in place a GMAC root cannot reach the root + * integrity check, so without these a regression there would leave the whole suite + * green. Defence in depth is only depth if the inner layer is held to its contract + * independently. */ @Test - void rootIntegrityRefusesGmacWhenCalledDirectly() { - assertThatThrownBy(() -> TDF.rootIntegrity(new byte[64], new byte[32], Config.IntegrityAlgorithm.GMAC)) + void rootAlgorithmGuardRefusesGmac() { + assertThatThrownBy(() -> TDF.requireSupportedRootIntegrityAlgorithm(Config.IntegrityAlgorithm.GMAC)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("unsupported root integrity algorithm"); } @Test - void rootIntegrityRefusesNullWhenCalledDirectly() { - assertThatThrownBy(() -> TDF.rootIntegrity(new byte[64], new byte[32], null)) + void rootAlgorithmGuardRefusesNull() { + assertThatThrownBy(() -> TDF.requireSupportedRootIntegrityAlgorithm(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("unsupported root integrity algorithm"); } @Test - void rootIntegrityAcceptsHs256() { - assertThat(TDF.rootIntegrity(new byte[64], new byte[32], Config.IntegrityAlgorithm.HS256)) - .hasSize(32); + void rootAlgorithmGuardAcceptsHs256() { + assertThatCode(() -> TDF.requireSupportedRootIntegrityAlgorithm(Config.IntegrityAlgorithm.HS256)) + .doesNotThrowAnyException(); } @Test - void segmentIntegrityRefusesNullWhenCalledDirectly() { - assertThatThrownBy(() -> TDF.segmentIntegrity(new byte[64], new byte[32], null)) + void segmentAlgorithmGuardRefusesNull() { + assertThatThrownBy(() -> TDF.requireSupportedSegmentIntegrityAlgorithm(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("unsupported segment integrity algorithm"); } diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java index 2f5d22fc..84433a88 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java @@ -5,13 +5,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.StringWriter; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class TDFWriterTest { @Test @@ -64,14 +69,39 @@ void simpleTDFCreate() throws IOException { " }\n" + "}"; String payload = "Hello, world!"; - FileOutputStream fileOutStream = new FileOutputStream("sample.tdf"); - TDFWriter writer = new TDFWriter(fileOutStream); - try (var p = writer.payload()) { - new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + var tdfFile = File.createTempFile("sample", ".tdf"); + tdfFile.deleteOnExit(); + try (var fileOutStream = new FileOutputStream(tdfFile)) { + TDFWriter writer = new TDFWriter(fileOutStream); + try (var p = writer.payload()) { + new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + } + try (var m = writer.manifest()) { + m.write(kManifestJsonFromTDF.getBytes(StandardCharsets.UTF_8)); + } + writer.finish(); + } + + // our own reader + try (var channel = FileChannel.open(tdfFile.toPath(), StandardOpenOption.READ)) { + var entries = new ZipReader(channel).getEntries().stream() + .collect(Collectors.toMap(ZipReader.Entry::getName, e -> e)); + assertEquals(kManifestJsonFromTDF, + new String(entries.get(TDFWriter.TDF_MANIFEST_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + assertEquals(payload, + new String(entries.get(TDFWriter.TDF_PAYLOAD_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + } + + // an independent central-directory based reader, as a stand-in for the other SDKs + try (var zipFile = new ZipFile(tdfFile)) { + var manifestEntry = zipFile.getEntry(TDFWriter.TDF_MANIFEST_FILE_NAME); + assertNotNull(manifestEntry); + try (var in = zipFile.getInputStream(manifestEntry)) { + assertEquals(kManifestJsonFromTDF, new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } } - writer.appendManifest(kManifestJsonFromTDF); - writer.finish(); - fileOutStream.close(); } /** @@ -89,12 +119,18 @@ void readsBackAManifestWrittenPastTheZip64Boundary() throws IOException { try (var p = writer.payload()) { new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); } - writer.appendManifest(manifest); + try (var m = writer.manifest()) { + m.write(manifest.getBytes(StandardCharsets.UTF_8)); + } writer.finish(); try (var chan = new SeekableInMemoryByteChannel(out.toByteArray())) { var reader = new TDFReader(chan); - assertEquals(manifest, reader.manifest()); + var readBack = new StringWriter(); + try (var m = reader.manifest()) { + m.transferTo(readBack); + } + assertEquals(manifest, readBack.toString()); var payloadBytes = new byte[payload.length()]; assertEquals(payload.length(), reader.readPayloadBytes(payloadBytes));