Skip to content

Commit 4e7597c

Browse files
Metehan-Bicerclaude
andcommitted
fix(fill): accept a surrogate pair split across stdin string chunks
- A string stream may split a surrogate pair across chunks, and the per-chunk lone-surrogate check refused valid input. A trailing high surrogate is now held back until the next string chunk; a byte chunk or the end of input still refuses it as invalid UTF-8. - Size a string chunk with Buffer.byteLength before encoding it, so an oversized chunk is refused without being copied first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 9d2bc90 commit 4e7597c

2 files changed

Lines changed: 50 additions & 7 deletions

File tree

‎src/commands/interaction/fill-text-stdin.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ test.each([
2929
['surrounding spaces and inner newlines', [' two\nlines \n'], ' two\nlines '],
3030
['a value split across chunks', ['hun', Buffer.from('ter'), '2\n'], 'hunter2'],
3131
['a multi-byte character split across chunks', [Buffer.from([0xc3]), Buffer.from([0xa7])], 'ç'],
32+
['a surrogate pair split across string chunks', ['a\uD83D', '\uDE00b'], 'a\u{1F600}b'],
3233
])('reads %s', async (_name, chunks, expected) => {
3334
expect(await readFillTextFromStdin(pipe(...chunks))).toBe(expected);
3435
});
@@ -64,6 +65,18 @@ test.each([
6465
'fill_text_stdin_invalid_utf8',
6566
'lone-surrogate-',
6667
],
68+
[
69+
'a high surrogate followed by a byte chunk',
70+
() => pipe('held-surrogate-\uD83D', Buffer.from('x')),
71+
'fill_text_stdin_invalid_utf8',
72+
'held-surrogate-',
73+
],
74+
[
75+
'an oversized string chunk before checking its surrogates',
76+
() => pipe(`${'s3cret'.repeat(FILL_TEXT_STDIN_MAX_BYTES)}\uDC00`),
77+
'fill_text_stdin_too_large',
78+
's3cret',
79+
],
6780
])('refuses %s with a typed reason and no echoed input', async (_name, stdin, reason, input) => {
6881
const error = await readError(stdin());
6982

‎src/commands/interaction/fill-text-stdin.ts‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { AppError } from '@agent-device/kernel/errors';
33
/** Most UTF-8 input `fill --text-stdin` accepts, trailing newline included. */
44
export const FILL_TEXT_STDIN_MAX_BYTES = 64 * 1024;
55

6-
type StdinSource = AsyncIterable<Uint8Array | string> & { isTTY?: boolean };
6+
type StdinSource = AsyncIterable<StdinPiece> & { isTTY?: boolean };
77

88
/**
99
* Reads the `fill --text-stdin` value. Exactly one trailing `\n` or `\r\n` is removed so a value
@@ -24,18 +24,21 @@ export async function readFillTextFromStdin(stdin: StdinSource): Promise<string>
2424
}
2525
const chunks: Uint8Array[] = [];
2626
let byteLength = 0;
27+
let heldHighSurrogate = '';
2728
for await (const chunk of stdin) {
28-
const bytes = typeof chunk === 'string' ? encodeWellFormed(chunk) : chunk;
29-
byteLength += bytes.byteLength;
29+
const next = joinHeldHighSurrogate(heldHighSurrogate, chunk);
30+
heldHighSurrogate = next.held;
31+
byteLength += utf8ByteLength(next.piece);
3032
if (byteLength > FILL_TEXT_STDIN_MAX_BYTES) {
3133
throw new AppError(
3234
'INVALID_ARGS',
3335
`fill --text-stdin accepts at most ${FILL_TEXT_STDIN_MAX_BYTES} bytes of input.`,
3436
{ reason: 'fill_text_stdin_too_large', maxBytes: FILL_TEXT_STDIN_MAX_BYTES },
3537
);
3638
}
37-
chunks.push(bytes);
39+
chunks.push(encodeWellFormed(next.piece));
3840
}
41+
if (heldHighSurrogate) throw invalidUtf8Error();
3942
const text = stripOneTrailingNewline(decodeUtf8(Buffer.concat(chunks)));
4043
if (text === '') {
4144
throw new AppError('INVALID_ARGS', 'fill --text-stdin received no text.', {
@@ -47,11 +50,38 @@ export async function readFillTextFromStdin(stdin: StdinSource): Promise<string>
4750
}
4851

4952
const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
53+
const TRAILING_HIGH_SURROGATE = /[\uD800-\uDBFF]$/;
54+
55+
type StdinPiece = Uint8Array | string;
56+
57+
/**
58+
* A string stream may split a surrogate pair across chunks, so a trailing high surrogate is held
59+
* back until the next string chunk; a byte chunk cannot complete it.
60+
*/
61+
function joinHeldHighSurrogate(
62+
held: string,
63+
chunk: StdinPiece,
64+
): { piece: StdinPiece; held: string } {
65+
if (typeof chunk !== 'string') {
66+
if (held) throw invalidUtf8Error();
67+
return { piece: chunk, held: '' };
68+
}
69+
const text = held + chunk;
70+
return TRAILING_HIGH_SURROGATE.test(text)
71+
? { piece: text.slice(0, -1), held: text.slice(-1) }
72+
: { piece: text, held: '' };
73+
}
74+
75+
/** Sized before encoding, so an oversized string chunk is refused without being copied. */
76+
function utf8ByteLength(piece: StdinPiece): number {
77+
return typeof piece === 'string' ? Buffer.byteLength(piece, 'utf8') : piece.byteLength;
78+
}
5079

5180
/** `Buffer.from` would turn a lone surrogate into U+FFFD; refuse it like an invalid byte. */
52-
function encodeWellFormed(chunk: string): Uint8Array {
53-
if (LONE_SURROGATE.test(chunk)) throw invalidUtf8Error();
54-
return Buffer.from(chunk, 'utf8');
81+
function encodeWellFormed(piece: StdinPiece): Uint8Array {
82+
if (typeof piece !== 'string') return piece;
83+
if (LONE_SURROGATE.test(piece)) throw invalidUtf8Error();
84+
return Buffer.from(piece, 'utf8');
5585
}
5686

5787
function decodeUtf8(bytes: Uint8Array): string {

0 commit comments

Comments
 (0)