diff --git a/apps/docs/openapi-v2-logs.json b/apps/docs/openapi-v2-logs.json index 4659fc1c390..52c8878d3e4 100644 --- a/apps/docs/openapi-v2-logs.json +++ b/apps/docs/openapi-v2-logs.json @@ -1451,7 +1451,7 @@ "contentWithheld": { "type": "boolean", "const": true, - "description": "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned." + "description": "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A withheld content field is omitted whole, never a key inside it: a content field absent from a span carrying this marker may exist on the run but was not returned, while a key missing inside a content field that is present is genuinely absent." }, "toolCalls": { "type": "array", diff --git a/apps/sim/lib/api/contracts/logs.ts b/apps/sim/lib/api/contracts/logs.ts index c975f1bce50..0c85e989a31 100644 --- a/apps/sim/lib/api/contracts/logs.ts +++ b/apps/sim/lib/api/contracts/logs.ts @@ -277,7 +277,7 @@ export const traceSpanSchema: z.ZodType = z contentWithheld: z .literal(true) .describe( - "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned." + "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A withheld content field is omitted whole, never a key inside it: a content field absent from a span carrying this marker may exist on the run but was not returned, while a key missing inside a content field that is present is genuinely absent." ) .optional(), toolCalls: z.array(toolCallSchema).describe('Tool calls recorded by the span.').optional(), diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 25be067e711..6c0bba8bd3c 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -313,7 +313,8 @@ export interface TraceSpan { * this span (`input`, `output`, `thinking`, `errorMessage`, or its tool-call and * provider-segment content), or reduced the span to structure only. Status, timing, * and usage remain. Without it a withheld field is - * indistinguishable from one the block never produced. + * indistinguishable from one the block never produced. A content field is omitted + * whole, so a key missing inside a field that is present is genuinely absent. */ contentWithheld?: true model?: string diff --git a/apps/sim/lib/mothership/agent-cli/engines.test.ts b/apps/sim/lib/mothership/agent-cli/engines.test.ts index 542b8d16119..dc5b575313d 100644 --- a/apps/sim/lib/mothership/agent-cli/engines.test.ts +++ b/apps/sim/lib/mothership/agent-cli/engines.test.ts @@ -147,19 +147,110 @@ describe('logs query on a span whose content was withheld', () => { expect(report.contentWithheld).toBeUndefined() }) - it('keeps a withheld run under --where instead of counting it as filtered out', async () => { + it('keeps a withheld run under --where as unchecked, not as a match or filtered out', async () => { const report = await logsQueryWithheld({ where: 'result.count=6' }) expect(report.filteredOut).toBe(0) expect(report.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1', 'run-2']) - expect(report.rows[1].note).toMatch(/withheld/) - expect(report.contentWithheld).toBe(1) + expect(report.rows[0].whereUnchecked).toBeUndefined() + expect(report.rows[1].whereUnchecked).toBe(true) + expect(report.whereUnchecked).toBe(1) }) - it('still resolves a readable --field when --where hits withheld content', async () => { - const report = await logsQueryWithheld({ field: 'duration', where: 'result.count=6' }) - expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 }) - expect(report.rows[1].note).toMatch(/--where/) - expect(report.contentWithheld).toBe(1) + it('counts an unchecked --where apart from a withheld --field', async () => { + const readableField = await logsQueryWithheld({ field: 'duration', where: 'result.count=6' }) + expect(readableField.rows[1]).toMatchObject({ runId: 'run-2', value: 12 }) + expect(readableField.rows[1].note).toMatch(/--where/) + expect(readableField.whereUnchecked).toBe(1) + expect(readableField.contentWithheld).toBeUndefined() + + const withheldField = await logsQueryWithheld({ where: 'result.count=6' }) + expect(withheldField.rows[1].note).toMatch(/value unknown/) + expect(withheldField.whereUnchecked).toBe(1) + expect(withheldField.contentWithheld).toBe(1) + }) +}) + +// The projection omits a content field whole, so a span marked for its `thinking` +// still carries an intact `output` whose missing keys are genuinely absent. +const PARTLY_WITHHELD_RUNS = { + [RUNS_PATH]: { + data: [ + { runId: 'run-1', status: 'completed' }, + { runId: 'run-2', status: 'completed' }, + ], + }, + '/api/v2/logs/run-1': { + data: { + traceSpans: [ + { + name: 'Router', + status: 'success', + output: { route: 'priority', thinking: 'not the omitted span thinking' }, + contentWithheld: true, + }, + ], + }, + }, + '/api/v2/logs/run-2': { + data: { + traceSpans: [ + { + name: 'Router', + status: 'success', + output: { content: 'no route chosen' }, + toolCalls: [{ name: 'lookup' }], + contentWithheld: true, + }, + ], + }, + }, +} + +async function logsQueryPartlyWithheld(flags: Record) { + const result = await runEngine('logs query', ['wf-1'], runtimeWith(PARTLY_WITHHELD_RUNS), { + block: 'Router', + ...flags, + }) + expect(result.exitCode).toBe(0) + return JSON.parse(result.stdout) +} + +describe('logs query on a marked span whose read field is intact', () => { + it('filters out a run whose intact output lacks the --where key', async () => { + const report = await logsQueryPartlyWithheld({ where: 'output.route=priority' }) + expect(report.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1']) + expect(report.filteredOut).toBe(1) + expect(report.whereUnchecked).toBeUndefined() + + const viaFallback = await logsQueryPartlyWithheld({ where: 'route=priority' }) + expect(viaFallback.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1']) + expect(viaFallback.filteredOut).toBe(1) + }) + + it('reports a key missing from intact output as not found, not withheld', async () => { + const report = await logsQueryPartlyWithheld({ field: 'output.route' }) + expect(report.rows[1]).toMatchObject({ + runId: 'run-2', + value: null, + note: 'path not found on span', + }) + expect(report.contentWithheld).toBeUndefined() + }) + + it('reports a path through an omitted field as withheld', async () => { + const thinking = await logsQueryPartlyWithheld({ field: 'thinking' }) + expect(thinking.rows[0].note).toMatch(/value unknown/) + expect(thinking.contentWithheld).toBe(2) + + const shadowed = await logsQueryPartlyWithheld({ field: 'thinking', where: 'route=priority' }) + expect(shadowed.rows[0]).toMatchObject({ runId: 'run-1', value: null }) + expect(shadowed.rows[0].note).toMatch(/value unknown/) + expect(shadowed.fieldResolvedUnder).toBeUndefined() + + const toolInput = await logsQueryPartlyWithheld({ field: 'toolCalls.0.input.query' }) + expect(toolInput.rows[1].note).toMatch(/value unknown/) + const toolMissing = await logsQueryPartlyWithheld({ field: 'toolCalls.3.input.query' }) + expect(toolMissing.rows[1].note).toBe('path not found on span') }) }) diff --git a/apps/sim/lib/mothership/agent-cli/engines/query.ts b/apps/sim/lib/mothership/agent-cli/engines/query.ts index 19c990f064b..724d73431ad 100644 --- a/apps/sim/lib/mothership/agent-cli/engines/query.ts +++ b/apps/sim/lib/mothership/agent-cli/engines/query.ts @@ -24,12 +24,33 @@ const TRACE_FETCH_CONCURRENCY = 5 const VALUE_MAX_CHARS = 600 const PATH_NOT_FOUND_NOTE = 'path not found on span' /** - * The log projection withheld this span's content rather than return what it could not - * prove safe to display, so an unresolved content path says nothing about the run. + * The log projection withheld the content field this path reads through rather than + * return what it could not prove safe to display, so the value says nothing about the run. */ const CONTENT_WITHHELD_NOTE = 'span content withheld by the log projection; value unknown' const WHERE_WITHHELD_NOTE = - 'span content withheld by the log projection; --where could not be checked, run kept' + 'span content withheld by the log projection; --where could not be checked, run kept but not a confirmed match' + +/** + * Where the log projection can omit content on a span. It omits a content field whole + * (never a key inside one), so a path is withheld only when it runs through one of these + * fields and the field itself is absent on a span carrying `contentWithheld`; a present + * field missing a nested key is a genuine miss. `#` matches an array index. + */ +const WITHHELD_CONTENT_PATHS: ReadonlyArray = [ + ['input'], + ['output'], + ['thinking'], + ['errorMessage'], + ['toolCalls', '#', 'input'], + ['toolCalls', '#', 'output'], + ['toolCalls', '#', 'error'], + ['modelToolCalls', '#', 'arguments'], + ['providerTiming', 'segments', '#', 'assistantContent'], + ['providerTiming', 'segments', '#', 'thinkingContent'], + ['providerTiming', 'segments', '#', 'errorMessage'], + ['providerTiming', 'segments', '#', 'toolCalls', '#', 'arguments'], +] /** * Keys that live on the span itself. A path whose head is anything else is read @@ -80,6 +101,8 @@ interface ResolvedSpanPath { /** The path that produced `value`: as given, or prefixed when the fallback applied. */ path: string resolvedUnder?: 'output' + /** Unresolved because the projection withheld the content field the path reads through. */ + withheld?: true } function collectSpansByName( @@ -102,16 +125,47 @@ function resolveSpanPath(span: QueryTraceSpan, path: string): unknown { return current } -/** The path as given first; under `output.` when the head is not a span-level key. */ +function isWithheldContentPrefix(segments: readonly string[]): boolean { + return WITHHELD_CONTENT_PATHS.some( + (pattern) => + pattern.length === segments.length && + pattern.every((part, index) => + part === '#' ? /^\d+$/.test(segments[index] ?? '') : part === segments[index] + ) + ) +} + +/** Whether an unresolved path stopped at a content field the projection omitted. */ +function isWithheldPath(span: QueryTraceSpan, path: string): boolean { + if (span.contentWithheld !== true) return false + const segments = path.split('.') + let current: unknown = span + for (let index = 0; index < segments.length; index++) { + if (current == null || typeof current !== 'object') return false + const next = (current as Record)[segments[index] ?? ''] + if (next === undefined) return isWithheldContentPrefix(segments.slice(0, index + 1)) + current = next + } + return false +} + +/** + * The path as given first; under `output.` when the head is not a span-level key. A + * direct path through a withheld content field (`thinking`) stays unknown rather than + * falling back to a same-named key under output. + */ function resolveSpanPathWithFallback(span: QueryTraceSpan, path: string): ResolvedSpanPath { const direct = resolveSpanPath(span, path) if (direct !== undefined) return { value: direct, path } + if (isWithheldPath(span, path)) return { value: undefined, path, withheld: true } const head = path.split('.')[0] ?? '' if (SPAN_LEVEL_KEYS.has(head)) return { value: undefined, path } const underOutput = `output.${path}` const value = resolveSpanPath(span, underOutput) - if (value === undefined) return { value: undefined, path } - return { value, path: underOutput, resolvedUnder: 'output' } + if (value !== undefined) return { value, path: underOutput, resolvedUnder: 'output' } + return isWithheldPath(span, underOutput) + ? { value: undefined, path, withheld: true } + : { value: undefined, path } } function clipValue(value: unknown): unknown { @@ -134,7 +188,10 @@ export const logsQueryCommand: AgentCliEngine = { return agentCliFail( 'Usage: sim logs query --block [--field ] [--where =] [--status ] [--trigger ] [--limit N]\n' + 'Paths resolve inside the matched block span: output.content, input.action_id, status, duration. ' + - 'A path that names no span-level key is read under output. (result.count means output.result.count).' + 'A path that names no span-level key is read under output. (result.count means output.result.count).\n' + + 'When the log projection withheld a content field (input, output, thinking, errorMessage, tool-call or provider-segment content), ' + + 'a path through it reports "value unknown" and, under --where, the run is kept with whereUnchecked: true (counted in whereUnchecked, not a confirmed match). ' + + 'A path missing inside a field that is present is "path not found" and fails --where.' ) } const field = stringFlag(flags, 'field') ?? 'output' @@ -167,6 +224,7 @@ export const logsQueryCommand: AgentCliEngine = { let filteredOut = 0 let missingTrace = 0 let contentWithheld = 0 + let whereUnchecked = 0 let fieldResolvedUnder: 'output' | undefined let whereResolvedUnder: 'output' | undefined const rows = await mapWithConcurrency( @@ -212,7 +270,7 @@ export const logsQueryCommand: AgentCliEngine = { const matched = resolveSpanPathWithFallback(last, wherePath) if (matched.resolvedUnder) whereResolvedUnder = matched.resolvedUnder // Filtering on a withheld value would silently drop a run that may well match. - whereWithheld = matched.value === undefined && last.contentWithheld === true + whereWithheld = matched.withheld === true if (!whereWithheld && String(matched.value) !== whereValue) { filteredOut++ return undefined @@ -220,8 +278,9 @@ export const logsQueryCommand: AgentCliEngine = { } const resolved = resolveSpanPathWithFallback(last, field) if (resolved.resolvedUnder) fieldResolvedUnder = resolved.resolvedUnder - const fieldWithheld = resolved.value === undefined && last.contentWithheld === true - if (whereWithheld || fieldWithheld) contentWithheld++ + const fieldWithheld = resolved.withheld === true + if (fieldWithheld) contentWithheld++ + if (whereWithheld) whereUnchecked++ const note = fieldWithheld ? CONTENT_WITHHELD_NOTE : resolved.value === undefined @@ -229,7 +288,12 @@ export const logsQueryCommand: AgentCliEngine = { : whereWithheld ? WHERE_WITHHELD_NOTE : undefined - return { ...matchedRow, value: clipValue(resolved.value), ...(note ? { note } : {}) } + return { + ...matchedRow, + value: clipValue(resolved.value), + ...(note ? { note } : {}), + ...(whereWithheld ? { whereUnchecked: true } : {}), + } } ) @@ -247,6 +311,7 @@ export const logsQueryCommand: AgentCliEngine = { where: whereResolvedUnder ? `output.${where}` : where, ...(whereResolvedUnder ? { whereResolvedUnder } : {}), filteredOut, + ...(whereUnchecked ? { whereUnchecked } : {}), } : {}), ...(missingTrace ? { missingTrace } : {}),