From c7fb620ac4006a06f1c24a09493b4ef7cf08ec67 Mon Sep 17 00:00:00 2001 From: Tom Shore Date: Sat, 10 Oct 2026 10:08:08 +0100 Subject: [PATCH 1/2] Hadoop uses a blockHeaderSize of 8 instead of the normal 4 --- .../xerial/snappy/SnappyHadoopCompatibleOutputStream.java | 5 +++++ src/main/java/org/xerial/snappy/SnappyOutputStream.java | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java b/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java index d9f69bf4..1857450d 100644 --- a/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java @@ -27,4 +27,9 @@ protected void writeBlockPreemble() { writeCurrentDataSize(); } + + @Override + protected int blockHeaderSize() { + return 8; + } } diff --git a/src/main/java/org/xerial/snappy/SnappyOutputStream.java b/src/main/java/org/xerial/snappy/SnappyOutputStream.java index 93b26e3b..1c6c97a0 100755 --- a/src/main/java/org/xerial/snappy/SnappyOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyOutputStream.java @@ -275,7 +275,7 @@ public void write(short[] f) private boolean hasSufficientOutputBufferFor(int inputSize) { int maxCompressedSize = Snappy.maxCompressedLength(inputSize); - return maxCompressedSize < outputBuffer.length - outputCursor - 4; + return maxCompressedSize < outputBuffer.length - outputCursor - blockHeaderSize(); } /** @@ -411,6 +411,10 @@ protected void writeCurrentDataSize(){ writeInt(outputBuffer, outputCursor, inputCursor); outputCursor += 4; } + + protected int blockHeaderSize(){ + return 4; + } /** * close the stream From b961bbcd0fcc60a5c2d3b4c759f50f35a8bc79df Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Sat, 10 Oct 2026 07:30:21 -0700 Subject: [PATCH 2/2] Add regression test for Hadoop block header overflow (#767) Writes a full incompressible block followed by trailing blocks of every size and decodes the Hadoop block format, covering the case where the output buffer had only 4-8 bytes of slack for the 8-byte block header. Co-Authored-By: Claude Opus 5.5 --- .../org/xerial/snappy/SnappyOutputStream.java | 2 +- ...HadoopCompatibleOutputStreamBlockTest.java | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamBlockTest.java diff --git a/src/main/java/org/xerial/snappy/SnappyOutputStream.java b/src/main/java/org/xerial/snappy/SnappyOutputStream.java index 1c6c97a0..9f65cdfe 100755 --- a/src/main/java/org/xerial/snappy/SnappyOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyOutputStream.java @@ -411,7 +411,7 @@ protected void writeCurrentDataSize(){ writeInt(outputBuffer, outputCursor, inputCursor); outputCursor += 4; } - + protected int blockHeaderSize(){ return 4; } diff --git a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamBlockTest.java b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamBlockTest.java new file mode 100644 index 00000000..f54cddca --- /dev/null +++ b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamBlockTest.java @@ -0,0 +1,54 @@ +package org.xerial.snappy; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; + +/** + * Regression test for https://github.com/xerial/snappy-java/issues/767: the Hadoop block preamble + * (raw length + compressed length = 8 bytes) must be reserved before compressing a block. + */ +public class SnappyHadoopCompatibleOutputStreamBlockTest +{ + @Test + public void incompressibleTrailingBlock() + throws Exception + { + int blockSize = SnappyOutputStream.DEFAULT_BLOCK_SIZE; + byte[] data = new byte[2 * blockSize]; + new Random(0).nextBytes(data); + // A full incompressible block followed by a trailing block of every size: some tail sizes leave + // between 4 and 8 bytes of slack in the output buffer, which used to overflow it. + for (int tail = 1; tail < blockSize; tail++) { + byte[] input = Arrays.copyOf(data, blockSize + tail); + + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + SnappyHadoopCompatibleOutputStream os = new SnappyHadoopCompatibleOutputStream(compressed, blockSize); + os.write(input); + os.close(); + + Assert.assertArrayEquals("tail=" + tail, input, decodeHadoopBlocks(compressed.toByteArray())); + } + } + + private static byte[] decodeHadoopBlocks(byte[] data) + throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int pos = 0; + while (pos < data.length) { + int rawLength = SnappyOutputStream.readInt(data, pos); + int compressedLength = SnappyOutputStream.readInt(data, pos + 4); + pos += 8; + byte[] raw = Snappy.uncompress(Arrays.copyOfRange(data, pos, pos + compressedLength)); + Assert.assertEquals(rawLength, raw.length); + out.write(raw); + pos += compressedLength; + } + return out.toByteArray(); + } +}