Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/agents/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ Before creating fixtures, look in `@agent-device/test-kit` and
`packages/daemon/src/__tests__/support/`. Shared `DeviceInfo`, session, snapshot, store,
runtime-fact, and mocked-binary values belong in a sibling fixture module, not in repeated test
literals; a fixture one package uses is a `*.fixtures.ts` beside its module. The packages test-kit
builds on (kernel, contracts, capture-kit, host-kit, command-registry, daemon-contracts) cannot
builds on (kernel, contracts, capture-kit, host-kit, daemon-contracts) cannot
declare it; their tests keep helpers local. Never import test-kit in the `vi.mock` factory of a
module its entry loads (`host-kit/process`, `command`, `file`): spy it in place, as
`pinOwnProcessStartTime` does. Fixtures a package still exports (R83's legacy count) stay
Expand Down
2 changes: 1 addition & 1 deletion packages/ad-script/src/internal/__tests__/script.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -894,7 +894,7 @@ test('formatDivergenceActionLabel categorically drops fill/type text but keeps t
// The property test asserting "serializing a parsed script is a fixed point
// for generated scripts" stays at `src/commands/replay/ad-script-round-trip.test.ts`:
// its script generator (`replayScriptArb`) is derived from the root command
// catalog and selector grammar (`@agent-device/test-kit`),
// catalog and selector grammar (`src/commands/replay/replay-script.fixtures.ts`),
// which this package cannot import without an R11 package→root-src escape
// (#1478 P5 scoping dossier §5d). It exercises this package's exports via
// the `@agent-device/ad-script` specifier instead of duplicating the codec.
1 change: 0 additions & 1 deletion packages/test-kit/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
"description": "Private shared test helpers (temp dirs, device/session/snapshot fixtures, mocked processes, loopback servers) for the root and workspace-package test trees. One entry, named exports; test-only, never bundled, never importing root `src/`, `platform-*`, `provider-*` or the daemon.",
"dependencies": {
"@agent-device/capture-kit": "workspace:*",
"@agent-device/command-registry": "workspace:*",
"@agent-device/contracts": "workspace:*",
"@agent-device/daemon-contracts": "workspace:*",
"@agent-device/host-kit": "workspace:*",
Expand Down
1 change: 0 additions & 1 deletion packages/test-kit/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ export {
PROPERTY_RUNS_SMALL,
rawSnapshotNodesArb,
refArb,
replayScriptArb,
} from './property-arbitraries.ts';
export {
createUnavailableRuntimeFactsForTest,
Expand Down
231 changes: 0 additions & 231 deletions packages/test-kit/src/property-arbitraries.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { SCROLL_DIRECTIONS, SWIPE_PRESETS } from '@agent-device/contracts/scroll-gesture';
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import fc from 'fast-check';
import { INTERNAL_COMMANDS, PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';

/**
* Shared fast-check generators for the pure parse/print and geometry kernels
Expand Down Expand Up @@ -74,232 +72,3 @@ export const rawSnapshotNodesArb: fc.Arbitrary<RawSnapshotNode[]> = fc
{ maxLength: 12 },
)
.map((nodes) => nodes.map((node, index) => ({ ...node, index })));

// ---------------------------------------------------------------------------
// `.ad` script lines
// ---------------------------------------------------------------------------

const scriptTextArb: fc.Arbitrary<string> = fc.oneof(
fc.constantFrom('Submit', 'Log in', 'a "quoted" label', "it's fine", 'tab\tseparated', 'Ünïcøde'),
fc.string({ minLength: 1, maxLength: 10 }),
);

const scriptTargetArb: fc.Arbitrary<string> = fc.oneof(
refArb.map(formatRef),
scriptTextArb.map((text) => JSON.stringify(`label=${text}`)),
// #3197: a hand-written line quotes a selector with single quotes the way the
// shell does. The formatter always rewrites to the JSON spelling, so this tests
// the tokenizer's reading without expecting the single-quote form back.
fc.constantFrom('Sign in', 'Log in', 'Submit').map((label) => `'label=${JSON.stringify(label)}'`),
);

const coordinateArb = fc.integer({ min: 0, max: 1200 }).map(String);

/**
* The `wait` line tails the script grammar accepts after the target: its budget
* positional, and the capture-scope flags it declares (#3197). Both may appear on
* one line, positional before flags, which is the order the writer emits them in.
*/
const WAIT_TIMEOUT_ARB: fc.Arbitrary<string> = fc
.integer({ min: 100, max: 5000 })
.map((timeout) => ` ${timeout}`);

const WAIT_CAPTURE_FLAG_ARB: fc.Arbitrary<string> = fc.oneof(
fc.constant(' --raw'),
fc.integer({ min: 0, max: 6 }).map((depth) => ` --depth ${depth}`),
scriptTextArb.map((scope) => ` --scope ${JSON.stringify(scope)}`),
);

/** The `scroll` flags the script grammar carries (#3197): the stop condition. */
const SCROLL_FLAG_ARB: fc.Arbitrary<string> = scriptTargetArb.map((target) => ` --until ${target}`);

/**
* A command is either generated from a line template or explicitly waived with
* the reason it needs none. `Record<PublicCommandName, …>` makes the
* classification exhaustive: a new public command does not compile until it is
* given a template or a waiver.
*/
/**
* Every command name a `.ad` line can carry. Internal commands are included
* because scripts record them too (`runtime` has its own parse/print branch).
*/
type ReplayCommandName =
| (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]
| (typeof INTERNAL_COMMANDS)[keyof typeof INTERNAL_COMMANDS];

type ReplayLinePlan = fc.Arbitrary<string> | { waived: string };

/**
* Commands whose `.ad` line is a bare `<command> <token>…` handled by the
* generic parse/print branch (`appendGenericActionScriptArgs`), whose shape the
* `wait`/`longpress` templates already exercise. A command that grows its own
* branch in the replay command's script formatting must move to a template.
*/
const GENERIC_REPLAY_LINE = {
waived: 'generic line shape, covered by the wait/longpress templates',
} as const;

const REPLAY_SCRIPT_LINE_PLANS = {
click: fc.oneof(
scriptTargetArb.map((target) => `click ${target}`),
fc.tuple(coordinateArb, coordinateArb).map(([x, y]) => `click ${x} ${y}`),
fc
.tuple(scriptTargetArb, fc.integer({ min: 1, max: 5 }))
.map(([target, count]) => `click ${target} --count ${count}`),
),
press: fc.oneof(
scriptTargetArb.map((target) => `press ${target}`),
fc
.tuple(scriptTargetArb, fc.constantFrom('primary', 'secondary', 'middle'))
.map(([target, button]) => `press ${target} --button ${button}`),
),
longpress: scriptTargetArb.map((target) => `longpress ${target}`),
hover: scriptTargetArb.map((target) => `hover ${target}`),
wait: fc
.tuple(
scriptTargetArb,
fc.option(WAIT_TIMEOUT_ARB, { nil: '' }),
fc.option(WAIT_CAPTURE_FLAG_ARB, { nil: '' }),
)
.map(([target, timeout, capture]) => `wait ${target}${timeout}${capture}`),
fill: fc
.tuple(scriptTargetArb, scriptTextArb)
.map(([target, text]) => `fill ${target} ${JSON.stringify(text)}`),
type: scriptTextArb.map((text) => `type ${JSON.stringify(text)}`),
get: fc
.tuple(fc.constantFrom('text', 'value', 'label'), scriptTargetArb)
.map(([property, target]) => `get ${property} ${target}`),
swipe: fc
.tuple(coordinateArb, coordinateArb, coordinateArb, coordinateArb)
.map((coordinates) => `swipe ${coordinates.join(' ')}`),
gesture: fc.oneof(
fc
.tuple(coordinateArb, coordinateArb, coordinateArb, coordinateArb)
.map((coordinates) => `gesture pan ${coordinates.join(' ')}`),
fc
.tuple(fc.constantFrom(...SCROLL_DIRECTIONS), coordinateArb, coordinateArb)
.map(([direction, x, y]) => `gesture fling ${direction} ${x} ${y}`),
fc.constantFrom(...SWIPE_PRESETS).map((preset) => `gesture swipe ${preset}`),
fc.integer({ min: -180, max: 180 }).map((degrees) => `gesture rotate ${degrees}`),
),
snapshot: fc.oneof(
fc.constant('snapshot'),
fc.constant('snapshot -i'),
fc.integer({ min: 0, max: 6 }).map((depth) => `snapshot -d ${depth}`),
scriptTextArb.map((scope) => `snapshot -s ${JSON.stringify(scope)}`),
),
screenshot: fc.constantFrom('screenshot', 'screenshot out.png'),
open: fc.oneof(
fc.constantFrom('open com.example.app', 'open com.example.app --relaunch'),
fc
.tuple(fc.constantFrom('ios', 'android'), fc.integer({ min: 1, max: 65_535 }))
.map(
([platform, port]) => `open com.example.app --platform ${platform} --metro-port ${port}`,
),
),
runtime: fc.oneof(
fc.constant('runtime metro'),
fc.constantFrom('ios', 'android').map((platform) => `runtime metro --platform ${platform}`),
),
record: fc.oneof(
fc.constantFrom('record start', 'record stop', 'record start --hide-touches'),
fc.integer({ min: 1, max: 120 }).map((fps) => `record start --fps ${fps}`),
),
alert: GENERIC_REPLAY_LINE,
'app-switcher': GENERIC_REPLAY_LINE,
'action-button': GENERIC_REPLAY_LINE,
fold: GENERIC_REPLAY_LINE,
apps: GENERIC_REPLAY_LINE,
appstate: GENERIC_REPLAY_LINE,
artifacts: GENERIC_REPLAY_LINE,
audio: GENERIC_REPLAY_LINE,
back: GENERIC_REPLAY_LINE,
batch: GENERIC_REPLAY_LINE,
boot: GENERIC_REPLAY_LINE,
capabilities: GENERIC_REPLAY_LINE,
clipboard: GENERIC_REPLAY_LINE,
close: GENERIC_REPLAY_LINE,
devices: GENERIC_REPLAY_LINE,
diff: GENERIC_REPLAY_LINE,
doctor: GENERIC_REPLAY_LINE,
events: GENERIC_REPLAY_LINE,
find: GENERIC_REPLAY_LINE,
focus: GENERIC_REPLAY_LINE,
home: GENERIC_REPLAY_LINE,
install: GENERIC_REPLAY_LINE,
'install-from-source': GENERIC_REPLAY_LINE,
is: GENERIC_REPLAY_LINE,
keyboard: GENERIC_REPLAY_LINE,
logs: GENERIC_REPLAY_LINE,
network: GENERIC_REPLAY_LINE,
orientation: GENERIC_REPLAY_LINE,
perf: GENERIC_REPLAY_LINE,
prepare: GENERIC_REPLAY_LINE,
push: GENERIC_REPLAY_LINE,
'react-native': GENERIC_REPLAY_LINE,
reinstall: GENERIC_REPLAY_LINE,
replay: GENERIC_REPLAY_LINE,
scroll: fc
.tuple(
fc.constantFrom(...SCROLL_DIRECTIONS),
fc.option(fc.constantFrom('0.5', '0.8'), { nil: undefined }),
fc.option(SCROLL_FLAG_ARB, { nil: undefined }),
)
.map(
([direction, amount, flags]) =>
`scroll ${direction}${amount === undefined ? '' : ` ${amount}`}${flags ?? ''}`,
),
settings: GENERIC_REPLAY_LINE,
shutdown: GENERIC_REPLAY_LINE,
test: GENERIC_REPLAY_LINE,
trace: GENERIC_REPLAY_LINE,
'trigger-app-event': GENERIC_REPLAY_LINE,
'tv-remote': GENERIC_REPLAY_LINE,
viewport: GENERIC_REPLAY_LINE,
human_control: { waived: 'host-local control commands are never recorded in replay scripts' },
install_source: GENERIC_REPLAY_LINE,
lease_allocate: GENERIC_REPLAY_LINE,
lease_heartbeat: GENERIC_REPLAY_LINE,
lease_release: GENERIC_REPLAY_LINE,
release_materialized_paths: GENERIC_REPLAY_LINE,
session_list: GENERIC_REPLAY_LINE,
session_save_script: GENERIC_REPLAY_LINE,
} satisfies Record<ReplayCommandName, ReplayLinePlan>;

/**
* Line templates for every command, resolved through the runtime command
* catalog so a command that exists at runtime but not in the plan fails loudly.
*/
function replayScriptLineArbs(): fc.Arbitrary<string>[] {
const arbitraries: fc.Arbitrary<string>[] = [];
const commands: ReplayCommandName[] = [
...Object.values(PUBLIC_COMMANDS),
...Object.values(INTERNAL_COMMANDS),
];
for (const command of commands) {
const plan: ReplayLinePlan | undefined = REPLAY_SCRIPT_LINE_PLANS[command];
if (plan === undefined) {
throw new Error(
`replay command "${command}" from @agent-device/command-registry/catalog is ` +
'unclassified: ' +
'add a line template or a waiver to REPLAY_SCRIPT_LINE_PLANS in ' +
'packages/test-kit/src/property-arbitraries.ts',
);
}
if (!('waived' in plan)) arbitraries.push(plan);
}
return arbitraries;
}

const replayScriptLineArb: fc.Arbitrary<string> = fc.oneof(...replayScriptLineArbs());

/** A valid `.ad` script: action lines interleaved with comments and blank lines. */
export const replayScriptArb: fc.Arbitrary<string> = fc
.array(
fc.oneof(
{ weight: 8, arbitrary: replayScriptLineArb },
{ weight: 1, arbitrary: fc.constantFrom('# a comment', '') },
),
{ minLength: 1, maxLength: 8 },
)
.map((lines) => `${lines.join('\n')}\n`);
1 change: 0 additions & 1 deletion packages/test-kit/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
"include": ["src"],
"references": [
{ "path": "../capture-kit" },
{ "path": "../command-registry" },
{ "path": "../contracts" },
{ "path": "../daemon-contracts" },
{ "path": "../host-kit" },
Expand Down
3 changes: 0 additions & 3 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion scripts/fuzz/arbitraries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// reaches deep parser branches that uniformly random noise never gets past the first token of.

import fc from 'fast-check';
import { replayScriptArb } from '@agent-device/test-kit';
import { replayScriptArb } from '../../src/commands/replay/replay-script.fixtures.ts';
Comment thread
thymikee marked this conversation as resolved.
import type { FuzzTarget, FuzzTargetName } from './target-types.ts';

const SELECTOR_VALUE_HAZARDS = [
Expand Down
1 change: 1 addition & 0 deletions scripts/fuzz/envelope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ export const CASE_GENERATION_INPUTS = [
'validation-arbitraries-maestro.ts',
'validation-values.ts',
'validation-case.ts',
'../../src/commands/replay/replay-script.fixtures.ts',
] as const;

/**
Expand Down
2 changes: 1 addition & 1 deletion src/commands/replay/ad-script-round-trip.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fc from 'fast-check';
import { test } from 'vitest';
import assert from 'node:assert/strict';
import { replayScriptArb } from '@agent-device/test-kit';
import { replayScriptArb } from './replay-script.fixtures.ts';
import { PROPERTY_RUNS } from '@agent-device/selectors/snapshot-geometry-fixtures';
import {
formatPortableActionLine,
Expand Down
Loading
Loading