From f50c1437111ec2dd4138bf4fd91f4397c78caf96 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 26 Oct 2026 19:07:28 +0100 Subject: [PATCH 1/2] test: expand fuzz coverage with Effect Arbitrary and generated data --- docs/testing.md | 20 +++- tests/fuzz/README.md | 37 ++++++- tests/fuzz/codecs.fuzz.test.ts | 104 ++++++++++++++++++ tests/fuzz/dialects/clickhouse.ts | 16 ++- tests/fuzz/dialects/postgres.ts | 29 +++-- tests/fuzz/dialects/sqlite.ts | 16 +-- tests/fuzz/fixtures.ts | 93 ++++++++++++++++ tests/fuzz/harness.test.ts | 50 +++++++++ tests/fuzz/harness.ts | 65 +++++++++--- tests/fuzz/literal-inventory.ts | 40 +++++++ tests/fuzz/literals.fuzz.test.ts | 22 ++-- tests/fuzz/migrations.fuzz.test.ts | 47 +++++++-- tests/fuzz/migrations.postgres.fuzz.test.ts | 111 ++++++++++++++++++++ tests/fuzz/model.ts | 66 +++++------- tests/fuzz/plan.ts | 14 +-- tests/fuzz/queries.fuzz.test.ts | 53 ++++++++-- tests/fuzz/registry.fuzz.test.ts | 29 +++++ tests/fuzz/regressions.ts | 21 +++- tests/fuzz/values.ts | 17 ++- 19 files changed, 737 insertions(+), 113 deletions(-) create mode 100644 tests/fuzz/codecs.fuzz.test.ts create mode 100644 tests/fuzz/fixtures.ts create mode 100644 tests/fuzz/harness.test.ts create mode 100644 tests/fuzz/literal-inventory.ts create mode 100644 tests/fuzz/migrations.postgres.fuzz.test.ts diff --git a/docs/testing.md b/docs/testing.md index 708b12e..e73c09f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -95,20 +95,28 @@ order, and `windowFunnel` with `strict_order` rejects intervening events. ## Fuzzing -`tests/fuzz/` generates random inputs with fast-check and checks them against oracles that do -not trust the builder: +`tests/fuzz/` generates schema-shaped inputs with Effect Arbitrary and SQL plans +with fast-check, checking them against independent oracles: -- `literals.fuzz.test.ts`: a random value of each column type reaches the database twice. One +- `literals.fuzz.test.ts`: a random value of each catalogued column type reaches each target twice. One copy goes through the database's own JSON parser as the reference, the other through the builder, both as a comparison literal and as a param in the SELECT list. The two must agree. - `queries.fuzz.test.ts`: random query plans (joins, nested conditions, grouping, having, - ordering, paging, subqueries) run on the core fixture and must match `model.ts`, a JS + ordering, paging, subqueries) run on the core fixture and generated tables and must match `model.ts`, a JS evaluator with SQL three-valued logic. A plan that reports `single-tenant` must have read one org's rows only. - `compile.fuzz.test.ts`: needs no database. Valid plans always compile, compile deterministically, leak no `undefined`, `NaN` or `__PARAM_`, and number Postgres placeholders `$1..$n`. Any JS value placed where a value goes is compiled or refused with a `QueryBuilderError`, never thrown and never a defect. +- `codecs.fuzz.test.ts`: Effect `TestSchema` verifies composed codec round trips; + schema-generated rows survive compiled encoding/decoding, and malformed wire rows + fail with a typed decode error. +- `migrations*.fuzz.test.ts`: SQLite/D1 and Postgres transitions are compared with + freshly created schemas and independent seed-row expectations, using varied text, + numeric, JSON, array and binary payloads as supported by each suite. +- `registry.fuzz.test.ts`: every dialect and public type export must have a literal + property or an explicit exemption. `bun run test` runs a small budget (a few seconds). Use `bun run fuzz` for longer runs: @@ -119,7 +127,9 @@ FUZZ_VERBOSE=1 bun run fuzz # print checked / skipped counts per propert ``` A failure prints the shrunk input, the SQL it compiled to, the database's reply, and a -`FUZZ_SEED=… FUZZ_PATH=…` command that replays it. Pin the input in +`FUZZ_SEED=… FUZZ_PATH=…` command that replays it. Effect-native properties print an +opaque token instead; use `FUZZ_REPLAY=…` with the failing test name. See [the fuzz guide](../tests/fuzz/README.md) for generator +choices and replay details. Pin the input in `tests/fuzz/regressions.ts`, which is replayed before any random input. Bugs found but not yet fixed live in `tests/fuzz/known-issues.ts`. Each entry recognises diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 99b2c40..e8197c3 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -1,6 +1,7 @@ # The fuzzer -Property-based tests for the query builder, run on real databases. They catch +Property-based tests for the query builder and codecs, run with Effect Arbitrary +and fast-check. Database properties run on real engines. They catch what hand-written cases miss: SQL that runs but returns the wrong rows, escaping bugs, and values the builder throws on instead of refusing. @@ -17,12 +18,40 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | File | Property | Oracle | | --- | --- | --- | -| `queries.fuzz.test.ts` | random plans, and `unionAll`s of them (branch and union ORDER BY / LIMIT / OFFSET), return the rows `model.ts` computes | a JS model of SQL, per target | +| `queries.fuzz.test.ts` | random plans and `unionAll`s, over both the core fixture and generated empty/duplicate/nullable/multi-tenant data, return the rows `model.ts` computes | a JS model of SQL, per target | | `writes.fuzz.test.ts` | random sequences of inserts (multi-row, defaulted cells, INSERT … SELECT, ON CONFLICT), updates and deletes, with RETURNING, leave a scratch table as the model says; unsupported writes are refused | a JS model of the table's rows | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | -| `migrations.fuzz.test.ts` | a SQLite migration between two random schemas (with table and column renames), run on seeded rows on node:sqlite and on D1 (one batch per migration), gives a fresh create's PRAGMA shape and keeps the rows; a drizzle-kit snapshot of any schema imports with nothing to generate | a fresh create, the seeded rows, and our own entities | -| `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | +| `migrations.fuzz.test.ts` | a SQLite migration between two random schemas (with table and column renames), run on seeded rows on node:sqlite and on D1 (one batch per migration), gives a fresh create's PRAGMA shape and keeps varied text, numeric and binary payloads; a drizzle-kit snapshot of any schema imports with nothing to generate | a fresh create, the seeded rows, and our own entities | +| `migrations.postgres.fuzz.test.ts` | random Postgres table/column/default/nullability/index transitions reach a fresh create's shape and preserve scalar, JSON and array rows | PGlite catalogs and independent seed values | +| `codecs.fuzz.test.ts` | schema-derived values round-trip through composed codecs and compiled row codecs; malformed wire rows fail with a typed decode error | Effect `TestSchema`, schema equivalence and explicit wire corruptions | +| `registry.fuzz.test.ts` | every dialect and public type export has fuzz coverage or an explicit reason | public exports and the literal catalog | + +## Choosing a generator + +Use Effect's native `Arbitrary.schema` for schema-shaped values: nested JSON, +UUIDs, booleans, brands and codec properties. `TestSchema.Asserts.verifyRoundTrip` +checks compositional encode/decode behavior. The shared harness accepts native +Effect arbitraries directly, preserving shrinking and replay; do not sample them +into fast-check constants, which loses their shrink tree. + +Keep fast-check's constrained generators for SQL domains, adversarial strings and +well-typed query/migration plans. A descriptor's wire codec can be broader than +its storage type: `UInt8.schema` accepts finite numbers, while valid SQL inputs +must be integers from 0 to 255. Schema-derived tests complement the independent +database/model oracles rather than replacing them. + +Both engines use `FUZZ_RUNS`, `FUZZ_SCALE` and `FUZZ_SEED`. Effect failures print an +opaque replay token; select the failing property and replay with: + +```sh +FUZZ_REPLAY='…' bun run fuzz -t 'compiled row codec rejects malformed wire \(postgres\)' +``` + +`FUZZ_PATH` belongs to fast-check; `FUZZ_REPLAY` belongs to Effect. Replay tokens +are version-specific. Pin important inputs in `regressions.ts` as well. A property +whose every input is discarded now fails, including known-issue skips; hitting a +configured time budget also fails rather than reporting partial coverage as success. ## Adding a dialect diff --git a/tests/fuzz/codecs.fuzz.test.ts b/tests/fuzz/codecs.fuzz.test.ts new file mode 100644 index 0000000..e44e8c6 --- /dev/null +++ b/tests/fuzz/codecs.fuzz.test.ts @@ -0,0 +1,104 @@ +// Schema properties exercise codec composition without a database. SQL-domain +// generators stay separate: a wire number schema is wider than an SQL integer. +import { Arbitrary, Effect, Exit, Schema } from "effect" +import { TestSchema } from "effect/testing" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as PG from "@maple-dev/effect-orm/postgres" +import * as SQLite from "@maple-dev/effect-orm/sqlite" +import { dialects } from "./dialects" +import { effectOptions, property, withContext } from "./harness" +import { normalize } from "./values" + +const boundedText = Schema.String.check(Schema.isMaxLength(32)) +const Id = Schema.NonEmptyString.check(Schema.isMaxLength(16)).pipe(Schema.brand("FuzzId")) +const payload = Schema.Struct({ + id: Id, + message: boundedText, + values: Schema.Array(Schema.NullOr(Schema.Int.check(Schema.isBetween({ minimum: -1000, maximum: 1000 })))).check(Schema.isMaxLength(6)), + metadata: Schema.Record(boundedText, Schema.NullOr(boundedText)), +}) +const composed = CH.array(CH.nullable(CH.brand(CH.string, Id))) +const transformed = CH.custom("String", Schema.FiniteFromString) +const cases = { + "ClickHouse numeric wire": CH.int64.schema, + "Postgres numeric wire": PG.int8.schema, + "ClickHouse nullable branded arrays": composed.schema, + "ClickHouse maps of arrays": CH.map(CH.string, CH.array(CH.nullable(CH.string))).schema, + "custom transformed values": transformed.schema, + "Postgres jsonb payload": PG.jsonb(payload).schema, + "SQLite json payload": SQLite.json(payload).schema, + "SQLite binary wire": SQLite.blob.schema, + "Postgres nested nullable arrays": PG.array(PG.array(PG.nullable(PG.text))).schema, +} satisfies Record> + +describe("Effect codec round-trips", () => { + for (const [name, schema] of Object.entries(cases)) { + it(`${name} survives encode and decode`, () => new TestSchema.Asserts(schema).verifyRoundTrip(effectOptions(100))) + } +}) + +const row = Schema.Struct({ id: Id, payload, transformed: Schema.FiniteFromString }) +const equivalentRows = Schema.toEquivalence(Schema.Array(row)) +const columns = { + id: CH.brand(CH.string, Id), + payload: CH.custom("String", Schema.fromJsonString(payload)), + transformed, +} + +describe("Effect generated compiled row codecs", () => { + for (const dialect of dialects) { + // These are wire-only codec tests; the custom descriptors need no SQL storage counterpart. + const table = dialect.module.table("codec_rows", { external: true, columns }) + const compiled = Effect.runSync(dialect.module.compile(dialect.module.from(table).select())) + const name = `compiled row codec retains generated values (${dialect.name})` + it(name, () => property(name, Arbitrary.schema(row), async (value) => { + const wire = await Effect.runPromise(compiled.encodeRows([value])) + const decoded = await Effect.runPromise(compiled.decodeRows(wire)) + await withContext({ wire, decoded }, () => { + expect(equivalentRows(decoded, [value])).toBe(true) + }) + }, { runs: 100 })) + + const invalidName = `compiled row codec rejects malformed wire (${dialect.name})` + it(invalidName, () => property(invalidName, Arbitrary.all({ + value: Arbitrary.schema(row), + corruption: Arbitrary.schema(Schema.Literals(["missing", "null", "json", "brand", "number"])), + }), async ({ value, corruption }) => { + const wire = await Effect.runPromise(compiled.encodeRows([value])) + const bad = { ...wire[0] } + switch (corruption) { + case "missing": delete bad.id; break + case "null": bad.payload = null; break + case "json": bad.payload = "{"; break + case "brand": bad.id = ""; break + case "number": bad.transformed = "not a number"; break + } + const exit = await Effect.runPromiseExit(compiled.decodeRows([wire[0]!, bad])) + await withContext({ corruption, bad, exit }, () => { + expect(Exit.isFailure(exit)).toBe(true) + if (Exit.isSuccess(exit)) return + expect(exit.cause.reasons).toHaveLength(1) + const reason = exit.cause.reasons[0]! + expect(reason._tag).toBe("Fail") + if (reason._tag !== "Fail") return + expect(reason.error._tag).toBe("@maple-dev/effect-orm/CompiledQueryDecodeError") + expect(reason.error.rowIndex).toBe(1) + }) + }, { runs: 100 })) + } +}) + +// A round trip uses the encoder's preferred union branch. Check alternate +// producer forms independently so quoted numeric wire values are covered too. +describe("Effect generated numeric wire representations", () => { + for (const [name, type] of [["clickhouse", CH.float64], ["postgres", PG.float8]] as const) { + const table = CH.table("numeric_rows", { external: true, columns: { value: type } }) + const compiled = CH.compileUnsafe(CH.from(table).select()) + const label = `quoted and unquoted numeric wire agree (${name})` + it(label, () => property(label, Arbitrary.schema(Schema.Finite), async (value) => { + const decoded = await Effect.runPromise(compiled.decodeRows([{ value }, { value: String(value) }])) + expect(normalize(decoded)).toEqual(normalize([{ value }, { value }])) + }, { runs: 100 })) + } +}) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index a93962b..4225924 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -1,9 +1,10 @@ // ClickHouse: live only when EFFECT_ORM_CLICKHOUSE_URL is set. Two targets, // because `join_use_nulls` changes what an unmatched LEFT JOIN row holds. import fc from "fast-check" +import { Arbitrary, Schema } from "effect" import * as CH from "@maple-dev/effect-orm/clickhouse" import { clickhouseContext } from "../../core-cases" -import { dateTimeWire, finite, int32, nastyString, nullOr, utc, type LiteralType } from "../values" +import { dateTimeWire, finite, int32, nastyString, nullOr, utc, wireDateTime, type LiteralType } from "../values" import { decode, type FuzzDialect, type FuzzTarget, type Outcome, type Semantics, type StringRules } from "./types" export const clickhouseEndpoint = process.env.EFFECT_ORM_CLICKHOUSE_URL @@ -101,7 +102,7 @@ const jsonSource = (sqlType: string) => (json: string) => const literal = ( id: string, type: CH.CHType, - value: fc.Arbitrary, + value: LiteralType["value"], options: { wire?: (value: unknown) => unknown; comparable?: boolean; source?: (value: unknown) => string } = {}, ): LiteralType => ({ id: `clickhouse:${id}`, @@ -115,8 +116,12 @@ const literal = ( const nasty = nastyString(strings) const literals: ReadonlyArray = [ literal("String", CH.string, nasty), + literal("brand(String)", CH.brand(CH.string, Schema.String.pipe(Schema.brand("FuzzText"))), nasty), + literal("custom(Int32)", CH.custom("Int32", CH.int32.schema), int32), literal("Int32", CH.int32, int32), literal("Int64", CH.int64, fc.maxSafeInteger()), + literal("UInt8", CH.uint8, fc.nat({ max: 255 })), + literal("UInt16", CH.uint16, fc.nat({ max: 65535 })), literal("UInt32", CH.uint32, fc.nat({ max: 2 ** 32 - 1 })), literal("UInt64", CH.uint64, fc.maxSafeNat()), // ClickHouse's text float parser (JSON input, `toFloat64('…')`) can be 1 ULP off unless @@ -128,11 +133,16 @@ const literals: ReadonlyArray = [ return `SELECT reinterpretAsFloat64(toUInt64(${view.getBigUint64(0)})) AS v` }, }), - literal("Bool", CH.bool, fc.boolean()), + literal("Bool", CH.bool, Arbitrary.schema(CH.bool.schema)), literal("DateTime", CH.dateTime, fc.integer({ min: MIN_DT / 1000, max: MAX_DT / 1000 }).map((s) => utc(s * 1000)), { wire: dateTimeWire(false), }), literal("DateTime64", CH.dateTime64, fc.integer({ min: MIN_DT64, max: MAX_DT64 }).map(utc), { wire: dateTimeWire(true) }), + literal("DateTimeString", CH.dateTimeString, fc.integer({ min: MIN_DT / 1000, max: MAX_DT / 1000 }).map((s) => wireDateTime(s * 1000, false))), + literal("DateTime64String", CH.dateTime64String, fc.integer({ min: MIN_DT64, max: MAX_DT64 }).map((ms) => wireDateTime(ms, true))), + literal("LowCardinality(String)", CH.lowCardinality(CH.string), nasty), + literal("SimpleAggregateFunction(max, Int64)", CH.simpleAggregateFunction("max", CH.int64), fc.maxSafeInteger()), + literal("DateTime64(0)", CH.precision(CH.dateTime64, 0), fc.integer({ min: MIN_DT64 / 1000, max: Math.floor(MAX_DT64 / 1000) }).map((s) => utc(s * 1000)), { wire: dateTimeWire(false) }), literal("Nullable(String)", CH.nullable(CH.string), nullOr(nasty)), literal("Nullable(Int64)", CH.nullable(CH.int64), nullOr(fc.maxSafeInteger())), literal("Array(String)", CH.array(CH.string), fc.array(nasty, { maxLength: 4 })), diff --git a/tests/fuzz/dialects/postgres.ts b/tests/fuzz/dialects/postgres.ts index cd5cb22..0227b35 100644 --- a/tests/fuzz/dialects/postgres.ts +++ b/tests/fuzz/dialects/postgres.ts @@ -1,10 +1,10 @@ // Postgres: an in-process PGlite, so it always runs. import fc from "fast-check" -import { DateTime } from "effect" +import { Arbitrary, DateTime, Schema } from "effect" import { PGlite } from "@electric-sql/pglite" import * as PG from "@maple-dev/effect-orm/postgres" import { postgresContext } from "../../core-cases" -import { finite, int32, nastyString, nullOr, utc, type LiteralType } from "../values" +import { finite, int32, jsonPayload, jsonPayloadSchema, nastyString, nullOr, utc, type LiteralType } from "../values" import { runWith, type FuzzDialect, type StringRules } from "./types" let pglite: PGlite | undefined @@ -29,27 +29,44 @@ const arraySource = (sqlType: string) => (text: string) => const literal = ( id: string, type: PG.PgType, - value: fc.Arbitrary, - options: { wire?: (value: unknown) => unknown; array?: boolean } = {}, + value: LiteralType["value"], + options: { wire?: (value: unknown) => unknown; array?: boolean; source?: (json: string) => string; normalizeValue?: (value: unknown) => unknown } = {}, ): LiteralType => ({ id: `postgres:${id}`, type, value, wire: options.wire ?? ((v) => v), - source: options.array ? arraySource(type.sql.replace(/\[\]$/, "")) : scalarSource(type.sql), + source: options.source ?? (options.array ? arraySource(type.sql.replace(/\[\]$/, "")) : scalarSource(type.sql)), comparable: true, + normalizeValue: options.normalizeValue, }) const nasty = nastyString(strings) const literals: ReadonlyArray = [ literal("text", PG.text, nasty), + literal("brand(text)", PG.brand(PG.text, Schema.String.pipe(Schema.brand("FuzzText"))), nasty), + literal("custom(int4)", PG.custom("int4", PG.int4.schema), int32), + literal("uuid", PG.uuid, Arbitrary.schema(Schema.String.check(Schema.isUUID())).pipe(Arbitrary.map((value) => value.toLowerCase()))), + literal("int2", PG.int2, fc.integer({ min: -(2 ** 15), max: 2 ** 15 - 1 })), literal("int4", PG.int4, int32), literal("int8", PG.int8, fc.maxSafeInteger()), + // Postgres prints float4's shortest round-trippable decimal, which a JS driver + // parses as float64. Compare in float32 precision, including subnormal/extreme values. + literal("float4", PG.float4, fc.float({ noNaN: true, noDefaultInfinity: true }), { + normalizeValue: (value) => Math.fround(value as number), + }), literal("float8", PG.float8, finite), - literal("boolean", PG.bool, fc.boolean()), + literal("numeric", PG.numeric, finite), + literal("boolean", PG.bool, Arbitrary.schema(PG.bool.schema)), literal("timestamptz", PG.timestamptz, fc.integer({ min: MIN_TS, max: MAX_TS }).map(utc), { wire: (value) => new Date(DateTime.toEpochMillis(value as DateTime.Utc)).toISOString(), }), + literal("timestamptzMillis", PG.timestamptzMillis, fc.integer({ min: MIN_TS, max: MAX_TS }), { + wire: (value) => new Date(value as number).toISOString(), + }), + literal("jsonb", PG.jsonb(jsonPayloadSchema), jsonPayload(strings), { + source: (text) => `SELECT ${json(text)}->'v' AS "v"`, + }), literal("nullable(text)", PG.nullable(PG.text), nullOr(nasty)), literal("nullable(int8)", PG.nullable(PG.int8), nullOr(fc.maxSafeInteger())), literal("text[]", PG.array(PG.text), fc.array(nasty, { maxLength: 4 }), { array: true }), diff --git a/tests/fuzz/dialects/sqlite.ts b/tests/fuzz/dialects/sqlite.ts index 459aa62..70aa0ee 100644 --- a/tests/fuzz/dialects/sqlite.ts +++ b/tests/fuzz/dialects/sqlite.ts @@ -2,11 +2,11 @@ // Miniflare). D1 executes SQLite's SQL, so both share semantics and literals; // they differ in the dialect they compile with and the database they run on. import fc from "fast-check" -import { DateTime, Schema } from "effect" +import { Arbitrary, DateTime, Schema } from "effect" import * as SQLite from "@maple-dev/effect-orm/sqlite" import { d1Context, sqliteContext } from "../../core-cases" import { d1Database, sqliteDatabase } from "../../sqlite-support" -import { finite, nastyString, nullOr, utc, type LiteralType } from "../values" +import { finite, jsonPayload, jsonPayloadSchema, nastyString, nullOr, utc, type LiteralType } from "../values" import { runWith, type DialectModule, type FuzzDialect, type Semantics, type StringRules, type WriteSupport } from "./types" const semantics: Semantics = { @@ -25,7 +25,7 @@ const semantics: Semantics = { // SQLite stores U+0000, but its drivers end text there, so the dialect refuses it. const strings: StringRules = { nul: false, leadingBom: true } // The years SQLite's text timestamps and date functions cover. -const MIN_TS = Date.UTC(0, 0, 1) +const MIN_TS = Date.parse("0000-01-01T00:00:00.000Z") const MAX_TS = Date.UTC(9999, 11, 31, 23, 59, 59, 999) const hex = (bytes: Uint8Array) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("") @@ -64,7 +64,7 @@ const literals = (dialect: string): ReadonlyArray => { const literal = ( id: string, type: SQLite.SqliteType, - value: fc.Arbitrary, + value: LiteralType["value"], options: { wire?: (value: unknown) => unknown; read?: (value: string) => string; source?: (value: unknown) => string; comparable?: boolean } = {}, ): LiteralType => ({ id: `${dialect}:${id}`, @@ -76,19 +76,21 @@ const literals = (dialect: string): ReadonlyArray => { }) return [ literal("text", SQLite.text, nasty), + literal("brand(text)", SQLite.brand(SQLite.text, Schema.String.pipe(Schema.brand("FuzzText"))), nasty), + literal("custom(text)", SQLite.custom("text", Schema.String), nasty), literal("integer", SQLite.integer, fc.maxSafeInteger()), literal("real", SQLite.real, finite, { source: (v) => exactReal(v as number) }), - literal("boolean", SQLite.boolean, fc.boolean()), + literal("boolean", SQLite.boolean, Arbitrary.schema(SQLite.boolean.schema)), // The reference row is SQLite's own text: datetime() writes the date and time, so a builder // value compares equal to what the database itself writes, not just to itself. - literal("timestamp", SQLite.timestamp, fc.integer({ min: MIN_TS, max: MAX_TS }).map(utc), { + literal("timestamp", SQLite.timestamp, fc.oneof(fc.constant(MIN_TS), fc.constant(Date.parse("0099-12-31T23:59:59.999Z")), fc.integer({ min: MIN_TS, max: MAX_TS })).map(utc), { source: (value) => { const ms = DateTime.toEpochMillis(value as DateTime.Utc) return `SELECT datetime(${Math.floor(ms / 1000)}, 'unixepoch') || '.' || printf('%03d', ${((ms % 1000) + 1000) % 1000}) AS "v"` }, }), literal("blob", SQLite.blob, fc.uint8Array({ maxLength: 8 }), { wire: (value) => hex(value as Uint8Array), read: (v) => `unhex(${v})` }), - literal("json", SQLite.json(Schema.Struct({ message: Schema.String })), fc.record({ message: nasty }), { + literal("json", SQLite.json(jsonPayloadSchema), jsonPayload(strings), { wire: (value) => JSON.stringify(value), comparable: false, }), diff --git a/tests/fuzz/fixtures.ts b/tests/fuzz/fixtures.ts new file mode 100644 index 0000000..c5a23aa --- /dev/null +++ b/tests/fuzz/fixtures.ts @@ -0,0 +1,93 @@ +// Independent, shrinkable input data for the query oracle and database CTEs. +import fc from "fast-check" +import type { CoreContext } from "../core-cases" +import { fixtureRows } from "../core-cases" +import type { FuzzDialect } from "./dialects/types" +import { wireDateTime } from "./values" + +export interface Order { + readonly OrgId: string + readonly Id: number + readonly Customer: string + readonly Amount: number + readonly Status: string + readonly Note: string | null + readonly Created: number +} +export interface Customer { + readonly OrgId: string + readonly Name: string + readonly Tier: string +} +export interface Fixture { + readonly orders: ReadonlyArray + readonly customers: ReadonlyArray +} + +export const coreFixture: Fixture = { + orders: fixtureRows.orders.map(([OrgId, Id, Customer, Amount, Status, Note, Created]) => ({ + OrgId: OrgId as string, Id: Id as number, Customer: Customer as string, Amount: Amount as number, + Status: Status as string, Note: Note as string | null, Created: Date.parse(`${(Created as string).replace(" ", "T")}Z`), + })), + customers: fixtureRows.customers.map(([OrgId, Name, Tier]) => ({ OrgId: OrgId as string, Name: Name as string, Tier: Tier as string })), +} + +const org = fc.constantFrom("org_1", "org_2", "org_3") +const name = fc.constantFrom("acme", "globex", "initech", "umbrella", "it's", "") +/** Small shared pools deliberately produce matching keys, duplicates and missing joins. */ +export const fixture: fc.Arbitrary = fc.record({ + orders: fc.array(fc.record({ + OrgId: org, + Customer: name, + Amount: fc.oneof(fc.constantFrom(0, 5, 7, 10, 20, 99), fc.integer({ min: -100, max: 120 })), + Status: fc.constantFrom("paid", "open", "void", ""), + Note: fc.option(fc.constantFrom("rush", "it's", "", "acme"), { nil: null }), + Created: fc.integer({ min: 0, max: 4 }).map((i) => Date.UTC(2026, 0, 1) + i * 600_000), + }), { maxLength: 10 }).map((rows) => rows.map((row, i) => ({ ...row, Id: i + 1 }))), + customers: fc.array(fc.record({ OrgId: org, Name: name, Tier: fc.constantFrom("gold", "silver", "bronze", "platinum", "") }), { maxLength: 8 }), +}) + +const columns = { + orders: [["OrgId", "String", "text"], ["Id", "UInt32", "int4"], ["Customer", "String", "text"], ["Amount", "Int64", "int8"], ["Status", "String", "text"], ["Note", "Nullable(String)", "text"], ["Created", "DateTime64(3, 'UTC')", "timestamptz"]], + customers: [["OrgId", "String", "text"], ["Name", "String", "text"], ["Tier", "String", "text"]], +} as const + +/** Values come from the restricted fixture pools, independently of the builder's literal writer. */ +export const fixtureSource = (dialect: string, data: Fixture, table: keyof Fixture): string => { + const defs = columns[table] + const value = (v: unknown): string => v === null ? "NULL" : typeof v === "number" ? String(v) : `'${String(v).replaceAll("'", "''")}'` + const select = (row: Order | Customer | undefined) => defs.map(([name, ch, pg]) => { + const raw = row === undefined ? null : (row as unknown as Record)[name] + const v = name === "Created" && raw !== null ? wireDateTime(raw as number, true) : raw + const type = dialect === "clickhouse" ? ch : dialect === "postgres" ? pg : name === "Id" || name === "Amount" ? "INTEGER" : "TEXT" + // An empty relation still needs valid values for non-nullable ClickHouse casts. + const empty = row === undefined && dialect === "clickhouse" ? (name === "Note" ? null : name === "Created" ? "2026-01-01 00:00:00.000" : type.startsWith("U") || type === "Int64" ? 0 : "") : v + return `CAST(${value(empty)} AS ${type}) AS "${name}"` + }).join(", ") + if (dialect === "clickhouse" && data[table].length > 0) { + // Match the core fixture's values() source. ClickHouse 26.2 incorrectly + // pushes DISTINCT/OFFSET into UNION ALL sources made from constant SELECTs. + const structure = defs.map(([name, type]) => `${name} ${type}`).join(", ") + const tuples = data[table].map((row) => `(${defs.map(([name]) => { + const raw = (row as unknown as Record)[name] + return value(name === "Created" ? wireDateTime(raw as number, true) : raw) + }).join(", ")})`).join(", ") + return `SELECT * FROM values('${structure.replaceAll("'", "\\'")}', ${tuples})` + } + if ((dialect === "sqlite" || dialect === "d1") && data[table].length > 0) { + // D1 caps compound SELECTs at five terms; VALUES has no such row limit. + const tuples = data[table].map((row) => `(${defs.map(([name]) => { + const raw = (row as unknown as Record)[name] + return value(name === "Created" ? wireDateTime(raw as number, true) : raw) + }).join(", ")})`).join(", ") + return `SELECT ${defs.map(([name], i) => `column${i + 1} AS "${name}"`).join(", ")} FROM (VALUES ${tuples})` + } + return data[table].length === 0 ? `SELECT ${select(undefined)} WHERE 1 = 0` : data[table].map((row) => `SELECT ${select(row)}`).join(" UNION ALL ") +} + +export const fixtureContext = (dialect: FuzzDialect, data: Fixture): CoreContext => ({ + ...dialect.context, + from: ((table) => dialect.module.from(table) + .withCTE("orders", fixtureSource(dialect.name, data, "orders")) + .withCTE("customers", fixtureSource(dialect.name, data, "customers"))) as CoreContext["from"], +}) diff --git a/tests/fuzz/harness.test.ts b/tests/fuzz/harness.test.ts new file mode 100644 index 0000000..ec6c872 --- /dev/null +++ b/tests/fuzz/harness.test.ts @@ -0,0 +1,50 @@ +import { Arbitrary, Schema } from "effect" +import fc from "fast-check" +import { afterEach, describe, expect, it } from "vitest" +import { config, FuzzFailure, property, skip, skipKnown } from "./harness" + +const original = { ...config } +afterEach(() => Object.assign(config, original)) +// Infrastructure checks use fixed budgets regardless of a soak's environment. +const reset = () => Object.assign(config, { runs: undefined, scale: 1, seed: 1, path: undefined, replay: undefined, verbose: false }) + +describe("fuzz harness coverage guarantees", () => { + it("refuses zero-run and invalid budgets for both generators", async () => { + reset() + config.runs = 0 + await expect(property("zero fast-check", fc.constant(1), () => {}, { runs: 1 })).rejects.toThrow("FUZZ_RUNS") + await expect(property("zero native", Arbitrary.Constant(1), () => {}, { runs: 1 })).rejects.toThrow("FUZZ_RUNS") + config.runs = undefined + config.scale = Number.NaN + await expect(property("invalid scale", fc.constant(1), () => {}, { runs: 1 })).rejects.toThrow("FUZZ_SCALE") + }) + it("fails when all inputs hit a known issue", async () => { + reset() + await expect(property("all known", fc.constant(1), () => skipKnown("open-bug"), { runs: 1 })).rejects.toThrow("every input was skipped (open-bug)") + }) + it("fails when all inputs exceed the oracle", async () => { + reset() + await expect(property("all outside oracle", fc.constant(1), () => skip("oracle limit"), { runs: 1 })).rejects.toThrow("every input was skipped (oracle limit)") + }) + it("fails rather than passing a partially interrupted budget", async () => { + reset() + await expect(property("interrupted budget", fc.constant(1), async () => { + await new Promise((resolve) => setTimeout(resolve, 10)) + }, { runs: 100, timeoutMs: 1 })).rejects.toThrow("interrupted budget") + }) + it("preserves native shrinking, failure context and replay", async () => { + reset() + const arb = Arbitrary.schema(Schema.Int) + let failure = "" + try { + await property("native replay", arb, (n) => { throw new FuzzFailure("deliberate", { n, sql: "SELECT bad" }) }, { runs: 5 }) + } catch (error) { failure = String(error) } + expect(failure).toContain("Shrunk input:") + expect(failure).toContain("SELECT bad") + const token = /FUZZ_REPLAY='([^']+)'/.exec(failure)?.[1] + expect(token).toBeDefined() + config.replay = token + await expect(property("native replay", arb, () => { throw new FuzzFailure("deliberate", {}) }, { runs: 5 })).rejects.toThrow("Property falsified") + await expect(property("native replay", arb, () => {}, { runs: 5 })).rejects.toThrow("PropertyPassed") + }) +}) diff --git a/tests/fuzz/harness.ts b/tests/fuzz/harness.ts index f2c9c65..0d132bf 100644 --- a/tests/fuzz/harness.ts +++ b/tests/fuzz/harness.ts @@ -12,19 +12,22 @@ // - `known` issues are skipped by name (see `known-issues.ts`); // `FUZZ_STRICT=1` turns them all back on, `FUZZ_STRICT=,` just those. import fc from "fast-check" +import { Arbitrary, Effect } from "effect" import { pinnedFor } from "./regressions" const int = (name: string): number | undefined => { const raw = process.env[name] - return raw === undefined || raw === "" ? undefined : Number.parseInt(raw, 10) + return raw === undefined || raw === "" ? undefined : Number(raw) } export const config = { /** Multiplies every property's own default run count. */ - scale: Number.parseFloat(process.env.FUZZ_SCALE ?? "1"), + scale: Number(process.env.FUZZ_SCALE ?? "1"), runs: int("FUZZ_RUNS"), seed: int("FUZZ_SEED"), path: process.env.FUZZ_PATH || undefined, + /** Native Effect Arbitrary replay token; separate from fast-check shrink paths. */ + replay: process.env.FUZZ_REPLAY || undefined, /** `1`: ignore every known issue. A comma list of ids: ignore just those (to verify one fix). */ strict: new Set((process.env.FUZZ_STRICT ?? "").split(",").filter(Boolean)), verbose: process.env.FUZZ_VERBOSE === "1", @@ -86,6 +89,15 @@ export interface PropertyOptions { readonly timeoutMs?: number } +const runCount = (runs: number): number => { + if (config.runs === undefined && (!Number.isFinite(config.scale) || config.scale <= 0)) { + throw new Error("FUZZ_SCALE must be a positive finite number") + } + const count = config.runs ?? Math.max(1, Math.round(runs * config.scale)) + if (!Number.isSafeInteger(count) || count < 1) throw new Error("FUZZ_RUNS must be a positive safe integer") + return count +} + /** * Check `predicate` over `arb`, throwing a report a person can act on. * @@ -94,12 +106,13 @@ export interface PropertyOptions { */ export const property = async ( name: string, - arb: fc.Arbitrary, + arb: fc.Arbitrary | Arbitrary.Arbitrary, predicate: (input: T) => void | Promise, options: PropertyOptions, ): Promise => { + if (Arbitrary.isArbitrary(arb)) return nativeProperty(name, arb, predicate, options) + if (config.replay !== undefined) throw new Error("FUZZ_REPLAY is for Effect properties; select one with -t") const skips = new Map() - const knownSkips = new Set() let checked = 0 const details = await fc.check( fc.asyncProperty(arb, async (input) => { @@ -109,18 +122,18 @@ export const property = async ( } catch (error) { if (error instanceof Skip) { skips.set(error.reason, (skips.get(error.reason) ?? 0) + 1) - if (error.known) knownSkips.add(error.reason) fc.pre(false) } throw error } }), { - numRuns: config.runs ?? Math.max(1, Math.round(options.runs * config.scale)), + numRuns: runCount(options.runs), ...(config.seed !== undefined ? { seed: config.seed } : {}), ...(config.path !== undefined ? { path: config.path } : {}), examples: pinnedFor(name).map((input) => [input as T]), interruptAfterTimeLimit: options.timeoutMs, + markInterruptAsFailure: true, maxSkipsPerRun: 1000, endOnFailure: false, }, @@ -129,13 +142,7 @@ export const property = async ( const skipped = [...skips].map(([reason, count]) => `${count}× ${reason}`).join(", ") report(`[fuzz] ${name}: ${checked} checked${skipped ? `, skipped ${skipped}` : ""}`) } - // fast-check fails a run whose every input was discarded; say why instead. if (checked === 0 && details.numSkips > 0 && details.counterexample === null) { - // A property an open bug covers entirely is the backlog doing its job, not a broken generator. - if (knownSkips.size > 0) { - report(`[fuzz] ${name}: every input hits a known issue (${[...knownSkips].join(", ")})`) - return - } throw new Error(`[fuzz] ${name}: every input was skipped (${[...skips.keys()].join(", ")})`) } if (!details.failed) return @@ -158,3 +165,37 @@ export const property = async ( ].join("\n"), ) } + +/** Shared budgets and replay for native properties, including TestSchema assertions. */ +export const effectOptions = (runs: number): Arbitrary.CheckOptions => ({ + runs: runCount(runs), + ...(config.seed !== undefined ? { seed: config.seed } : {}), + ...(config.replay !== undefined ? { replay: config.replay } : {}), +}) + +/** Native Effect generation retains its own shrinking and opaque replay token. */ +const nativeProperty = async ( + name: string, + arb: Arbitrary.Arbitrary, + predicate: (input: T) => void | Promise, + options: PropertyOptions, +): Promise => { + if (config.path !== undefined) throw new Error("FUZZ_PATH is for fast-check properties; select one with -t") + // Pinned examples are independent of the generator and must keep working after changes to it. + for (const input of pinnedFor(name)) await predicate(input as T) + const check = Arbitrary.checkEffect(arb, (input) => Effect.tryPromise({ + try: async () => { await predicate(input); return true }, + catch: (error) => error, + }), effectOptions(options.runs)) + const result = await Effect.runPromise(options.timeoutMs === undefined ? check : check.pipe(Effect.timeout(options.timeoutMs))) + if (config.verbose) report(`[fuzz] ${name}: ${"runs" in result ? result.runs : 0} checked (Effect Arbitrary)`) + if (result._tag === "Passed") return + const failure = Arbitrary.formatCheckFailure(result) + const context = result._tag === "Falsified" && result.failure._tag === "PropertyError" && result.failure.error instanceof FuzzFailure + ? describeContext(result.failure.error.context) : "" + const pattern = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + const replay = result._tag === "Falsified" + ? `FUZZ_REPLAY='${result.replay}' bun run fuzz -t '${pattern}'` + : "" + throw new Error(`[fuzz] ${name}\n${failure}\n${context}\n${replay}`) +} diff --git a/tests/fuzz/literal-inventory.ts b/tests/fuzz/literal-inventory.ts new file mode 100644 index 0000000..e7d2bf9 --- /dev/null +++ b/tests/fuzz/literal-inventory.ts @@ -0,0 +1,40 @@ +// Each public runtime export from the type modules is a literal case or an +// explicit non-literal exemption. Registry tests reject new omissions and stale entries. +export const literalCoverage = { + clickhouse: { + string: "String", uint8: "UInt8", uint16: "UInt16", uint32: "UInt32", uint64: "UInt64", + int32: "Int32", int64: "Int64", float64: "Float64", bool: "Bool", + dateTime: "DateTime", dateTime64: "DateTime64", dateTimeString: "DateTimeString", dateTime64String: "DateTime64String", + array: "Array(String)", nullable: "Nullable(String)", map: "Map(String, String)", + lowCardinality: "LowCardinality(String)", simpleAggregateFunction: "SimpleAggregateFunction(max, Int64)", + precision: "DateTime64(0)", brand: "brand(String)", custom: "custom(Int32)", + }, + postgres: { + text: "text", uuid: "uuid", bool: "boolean", int2: "int2", int4: "int4", int8: "int8", + float4: "float4", float8: "float8", numeric: "numeric", timestamptz: "timestamptz", timestamptzMillis: "timestamptzMillis", + jsonb: "jsonb", array: "text[]", nullable: "nullable(text)", brand: "brand(text)", custom: "custom(int4)", + }, + sqlite: { + text: "text", integer: "integer", real: "real", boolean: "boolean", timestamp: "timestamp", blob: "blob", + json: "json", nullable: "nullable(text)", brand: "brand(text)", custom: "custom(text)", + }, +} as const + +export const literalExemptions = { + clickhouse: { + CHNumber: "Wire codec; numeric literal cases exercise it with database-generated wire values.", + aggregateState: "Opaque aggregate state is consumed by Merge, never a client literal.", + untyped: "Caller-defined SQL with no value validation contract; custom has a representative typed extension case.", + }, + postgres: { + PgNumber: "Wire codec; numeric literal cases exercise it with database-generated wire values.", + PgTimestampLiteral: "Timestamp literal codec; timestamptz literal cases exercise it.", + parsePgArrayLiteral: "Driver wire-array parser, not a descriptor.", + pgTimestampToIso: "Timestamp parsing helper, not a descriptor.", + timestampLiteral: "Timestamp codec factory, not a descriptor.", + }, + sqlite: { + sqliteTimestampText: "Timestamp formatting helper, not a descriptor.", + timestampNow: "SQL default expression, not a descriptor.", + }, +} as const diff --git a/tests/fuzz/literals.fuzz.test.ts b/tests/fuzz/literals.fuzz.test.ts index 9f69e3d..2b7b7a1 100644 --- a/tests/fuzz/literals.fuzz.test.ts +++ b/tests/fuzz/literals.fuzz.test.ts @@ -16,7 +16,7 @@ import { afterAll, describe, expect, it } from "vitest" import { Effect, Exit } from "effect" import type * as CH from "@maple-dev/effect-orm/clickhouse" -import { closeAll, dialects, type FuzzDialect } from "./dialects" +import { closeAll, dialects, type FuzzDialect, type FuzzTarget } from "./dialects" import { FuzzFailure, property, skip, skipKnown, withContext } from "./harness" import { knownLiteralIssue, type LiteralPath } from "./known-issues" import { normalize, referenceJson, wellFormed, type LiteralType } from "./values" @@ -31,7 +31,7 @@ const build = (dialect: FuzzDialect, literal: LiteralType, value: unknown, path: return base.select(($: any) => ({ v: $.v })).where(($: any) => [value === null ? $.v.isNull() : $.v.eq(value)]) } -const roundTrip = async (dialect: FuzzDialect, literal: LiteralType, value: unknown, path: LiteralPath): Promise => { +const roundTrip = async (dialect: FuzzDialect, target: FuzzTarget, literal: LiteralType, value: unknown, path: LiteralPath): Promise => { // The reference row goes through JSON, which cannot carry a lone surrogate the way SQL text does. if (!wellFormed(value)) skip("lone surrogate") const known = knownLiteralIssue({ literal, value, path }) @@ -39,22 +39,26 @@ const roundTrip = async (dialect: FuzzDialect, literal: LiteralType, value: unkn const exit = Effect.runSyncExit(dialect.module.compile(build(dialect, literal, value, path), { p: value })) if (Exit.isFailure(exit)) throw new FuzzFailure("the builder refused a valid value", { value, cause: String(exit.cause) }) const compiled = exit.value as CH.CompiledQuery - const outcome = await dialect.targets[0]!.run(compiled) + const outcome = await target.run(compiled) await withContext({ sql: compiled.sql, parameters: compiled.parameters, outcome }, () => { expect(outcome._tag).toBe("Rows") if (outcome._tag !== "Rows") return const expected = path === "select" ? { v: value, p: value } : { v: value } - expect(normalize(outcome.rows)).toEqual([normalize(expected)]) + const canonical = (row: unknown) => normalize(literal.normalizeValue === undefined ? row : + Object.fromEntries(Object.entries(row as Record).map(([key, value]) => [key, literal.normalizeValue!(value)]))) + expect(outcome.rows.map(canonical)).toEqual([canonical(expected)]) }) } describe("literal round-trip", () => { for (const dialect of dialects) { - for (const literal of dialect.literals) { - for (const path of ["compare", "select"] as const) { - if (path === "compare" && !literal.comparable) continue - const name = `${literal.id} ${path}` - it.skipIf(!dialect.targets[0]!.available)(name, () => property(name, literal.value, (value) => roundTrip(dialect, literal, value, path), { runs: 40 }), 120_000) + for (const target of dialect.targets) { + for (const literal of dialect.literals) { + for (const path of ["compare", "select"] as const) { + if (path === "compare" && !literal.comparable) continue + const name = `${literal.id} ${path} (${target.name})` + it.skipIf(!target.available)(name, () => property(name, literal.value, (value) => roundTrip(dialect, target, literal, value, path), { runs: 40 }), 120_000) + } } } } diff --git a/tests/fuzz/migrations.fuzz.test.ts b/tests/fuzz/migrations.fuzz.test.ts index 882c80d..6c0f325 100644 --- a/tests/fuzz/migrations.fuzz.test.ts +++ b/tests/fuzz/migrations.fuzz.test.ts @@ -21,6 +21,7 @@ import * as SQLite from "@maple-dev/effect-orm/sqlite" import { diffSqliteSchemas, fromDrizzleSqliteSnapshot, renderSqliteOp, renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaEntity } from "@maple-dev/effect-orm/schema" import { retryTransport } from "../sqlite-support" import { FuzzFailure, property, skip, withContext } from "./harness" +import { nastyString, portableStrings, wellFormed } from "./values" // Schema plans @@ -120,12 +121,37 @@ const entitiesOf = (plan: SchemaPlan): ReadonlyArray => { // Rows -/** A value for row `i` of a column: distinct per row, so UNIQUE and key constraints in either schema hold. */ -const seedValue = (column: ColumnPlan, i: number): string | number => - column.type === "integer" ? i + 1 : column.type === "real" ? i + 0.5 : `v${i}` +interface Payload { + readonly text: string + readonly integer: number + readonly real: number + readonly bytes: Uint8Array +} + +const payload = fc.record({ + text: nastyString(portableStrings).filter(wellFormed), + integer: fc.oneof(fc.integer({ min: -(2 ** 31), max: 2 ** 31 - 4 }), fc.constantFrom(-Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER - 3)), + // Dyadic fractions remain exact through SQLite's SQL parser and uniqueness offsets. + real: fc.integer({ min: -1_000_000, max: 1_000_000 }).map((n) => n / 8), + bytes: fc.uint8Array({ maxLength: 16 }), +}) + +/** Keep each row unique even when its generated payload shrinks to an empty/zero value. */ +const seedValue = (column: ColumnPlan, i: number, next: ColumnPlan | undefined, data: Payload | undefined): unknown => { + // Type-changing migrations have affinity conversions; their independent oracle below compares text. + if (data === undefined || (next !== undefined && next.type !== column.type)) { + return column.type === "integer" ? i + 1 : column.type === "real" ? i + 0.5 : `v${i}` + } + switch (column.type) { + case "text": return `${i}:${data.text}` + case "integer": return data.integer + i + case "real": return data.real + i + case "blob": return new Uint8Array([i, ...data.bytes]) + } +} const sqlValue = (value: unknown): string => - value === null ? "NULL" : typeof value === "number" ? String(value) : `'${String(value).replaceAll("'", "''")}'` + value === null ? "NULL" : value instanceof Uint8Array ? `X'${Buffer.from(value).toString("hex")}'` : typeof value === "number" ? String(value) : `'${String(value).replaceAll("'", "''")}'` /** * Whether rows can exist in both schemas. A new NOT NULL column without a @@ -156,7 +182,7 @@ interface Seed { } /** Rows for `before` that `after` accepts: NULL only where the new column allows it or fills it with a default. */ -const seedRows = (before: SchemaPlan, after: SchemaPlan, count: number, nulls: ReadonlyArray): ReadonlyArray => +const seedRows = (before: SchemaPlan, after: SchemaPlan, count: number, nulls: ReadonlyArray, data?: Payload): ReadonlyArray => before.map((table) => { const next = after.find((t) => t.name === table.name) const n = next === undefined || rowsFit(table, next) ? count : 0 @@ -175,7 +201,7 @@ const seedRows = (before: SchemaPlan, after: SchemaPlan, count: number, nulls: R column.nullable && !keyed.has(column.name) && (later === undefined || later.nullable || (later.default !== undefined && later.default.k !== "now" && !uniqueLater(column.name))) - return [column.name, nullable && nulls[(i * 7 + j) % nulls.length] ? null : seedValue(column, i)] + return [column.name, nullable && nulls[(i * 7 + j) % nulls.length] ? null : seedValue(column, i, later, data)] }), ), ) @@ -326,9 +352,9 @@ const withRenames = (before: SchemaPlan, after: SchemaPlan, plan: RenamePlan | u // The property -type Input = readonly [SchemaPlan, SchemaPlan, number, ReadonlyArray, RenamePlan?] +type Input = readonly [SchemaPlan, SchemaPlan, number, ReadonlyArray, RenamePlan?, Payload?] -const migrates = async ([before, after, count, nulls, renamePlan]: Input, engines: () => Promise) => { +const migrates = async ([before, after, count, nulls, renamePlan, data]: Input, engines: () => Promise) => { const { renamed, hints: renameHints, table, column } = withRenames(before, after, renamePlan) const from = entitiesOf(before) const to = entitiesOf(renamed) @@ -351,7 +377,7 @@ const migrates = async ([before, after, count, nulls, renamePlan]: Input, engine throw new FuzzFailure("the diff needs more than every drop and rename confirmed", { renamed, missingHints: diff.missingHints, unsupported: diff.unsupported }) } const migration = diff.ops.flatMap(renderSqliteOp) - const seeds = seedRows(before, after, count, nulls) + const seeds = seedRows(before, after, count, nulls, data) const [fresh, migrated] = await engines() await fresh.exec(renderSqliteSchema(to)) @@ -373,7 +399,7 @@ const migrates = async ([before, after, count, nulls, renamePlan]: Input, engine expect(migratedShape).toEqual(freshShape) for (const { name, want, rows, next } of tables) { // A type change may convert a value (`'1'` to `1`); compared as text, it is the same value. - const text = (v: unknown) => (v === null || v === undefined ? null : String(v)) + const text = (v: unknown): unknown => v === null || v === undefined ? null : v instanceof Uint8Array ? [...v] : v instanceof ArrayBuffer ? [...new Uint8Array(v)] : Array.isArray(v) ? v : String(v) const expected = want.map((row) => Object.fromEntries(Object.entries(row).filter(([, v]) => v !== undefined).map(([k, v]) => [column(next.name, k), text(v)])), ) @@ -393,6 +419,7 @@ const input = fc.tuple( fc.integer({ min: 0, max: 3 }), fc.array(fc.boolean(), { minLength: 1, maxLength: 5 }), fc.record({ table: fc.boolean(), column: fc.option(fc.nat(), { nil: null }), caseOnly: fc.boolean() }), + payload, ) describe("SQLite migrations", () => { diff --git a/tests/fuzz/migrations.postgres.fuzz.test.ts b/tests/fuzz/migrations.postgres.fuzz.test.ts new file mode 100644 index 0000000..1ce63bd --- /dev/null +++ b/tests/fuzz/migrations.postgres.fuzz.test.ts @@ -0,0 +1,111 @@ +// Fresh-create and migrated Postgres schemas must agree, including retained rows. +import fc from "fast-check" +import { Effect } from "effect" +import { PGlite } from "@electric-sql/pglite" +import { afterAll, describe, expect, it } from "vitest" +import * as PG from "@maple-dev/effect-orm/postgres" +import { diffPgSchemas, pgEntitiesOf, renderPgOp, renderPgSchema } from "@maple-dev/effect-orm/schema" +import { FuzzFailure, property, withContext } from "./harness" +import { nastyString, portableStrings, wellFormed } from "./values" + +interface Column { + readonly name: string + readonly nullable: boolean + readonly default: boolean +} +interface Table { + readonly name: string + readonly columns: ReadonlyArray + readonly indexed: boolean + readonly unique: boolean +} +const columnNames = ["text", "number", "flag", "json", "array"] as const +const tablePlan = (name: string) => fc.record({ + name: fc.constant(name), + columns: fc.shuffledSubarray([...columnNames], { minLength: 1 }).chain((names) => fc.tuple(...names.map((name) => fc.record({ name: fc.constant(name), nullable: fc.boolean(), default: fc.boolean() })))), + indexed: fc.boolean(), + unique: fc.boolean(), +}) +const schemaPlan = fc.subarray(["t0", "t1"]).chain((names) => fc.tuple(...names.map(tablePlan))) +const text = nastyString(portableStrings).filter(wellFormed) +const input = fc.record({ before: schemaPlan, after: schemaPlan, text, count: fc.integer({ min: 0, max: 4 }), number: fc.integer({ min: -(2 ** 31), max: 2 ** 31 - 5 }) }) + +const quote = (value: string) => `'${value.replaceAll("'", "''")}'` +const defaults: Record = { text: "it's\\default", number: 7, flag: true, json: { default: [null, true] }, array: ["a", "b'c"] } +const types = { text: PG.text, number: PG.int4, flag: PG.bool, json: PG.jsonb(), array: PG.array(PG.text) } +const entitiesOf = (tables: ReadonlyArray) => Effect.runSync(pgEntitiesOf(tables.map((table) => PG.table(table.name, { + columns: { + id: PG.int4, + ...Object.fromEntries(table.columns.map((column) => { + const base = types[column.name as keyof typeof types] + return [column.name, PG.column(column.nullable ? PG.nullable(base) : base, (column.default ? { default: defaults[column.name] } : {}) as never)] + })), + }, + primaryKey: ["id"], + indexes: table.indexed ? [(table.unique ? PG.uniqueIndex : PG.index)(`${table.name}_idx`, ["id"])] : [], +})))) + +let db: PGlite | undefined +const getDatabase = () => (db ??= new PGlite()) +afterAll(async () => { await db?.close() }) + +const shape = async (database: PGlite) => ({ + columns: (await database.query(`SELECT table_name, column_name, data_type, udt_name, is_nullable, column_default FROM information_schema.columns WHERE table_schema = current_schema() ORDER BY table_name, column_name`)).rows, + indexes: (await database.query<{ tablename: string; indexname: string; indexdef: string }>(`SELECT tablename, indexname, indexdef FROM pg_indexes WHERE schemaname = current_schema() ORDER BY tablename, indexname`)).rows.map((row) => ({ ...row, indexdef: row.indexdef.replace(/(?:fresh|migrated)\./g, "") })), +}) +const wireValue = (column: string, value: unknown): string => { + if (value === null) return "NULL" + if (column === "json") return `${quote(JSON.stringify(value))}::jsonb` + if (column === "array") return `ARRAY[${(value as string[]).map(quote).join(", ")}]::text[]` + return typeof value === "string" ? quote(value) : String(value) +} + +describe("Postgres migrations", () => { + const name = "postgres schema transitions match fresh creation and retain rows" + it(name, () => property(name, input, async ({ before, after, text, count, number }) => { + const database = getDatabase() + const from = entitiesOf(before) + const to = entitiesOf(after) + const initial = diffPgSchemas(from, to) + const diff = diffPgSchemas(from, to, initial.missingHints) + const migration = diff.ops.flatMap(renderPgOp) + const seeds = before.map((table) => { + const next = after.find((next) => next.name === table.name) + // Postgres cannot fill a newly NOT NULL column without a default for existing rows. + const fits = next === undefined || next.columns.every((column) => table.columns.some((old) => old.name === column.name) || column.nullable || column.default) + return { table, rows: Array.from({ length: fits ? count : 0 }, (_, i) => ({ + id: i, + ...Object.fromEntries(table.columns.map((column) => [column.name, column.nullable && (next?.columns.find((next) => next.name === column.name)?.nullable ?? true) && i % 3 === 0 ? null : column.name === "text" ? `${i}:${text}` : column.name === "number" ? number + i : column.name === "flag" ? i % 2 === 0 : column.name === "json" ? { value: text, nested: [i, null, true] } : [text, String(i)]])), + })) } + }) + await withContext({ before, after, migration, seeds }, async () => { + expect(diff.unsupported).toEqual([]) + expect(diff.missingHints).toEqual([]) + await database.exec('DROP SCHEMA IF EXISTS fresh CASCADE; DROP SCHEMA IF EXISTS migrated CASCADE; CREATE SCHEMA fresh; CREATE SCHEMA migrated; SET search_path TO fresh') + for (const sql of renderPgSchema(to)) await database.exec(sql) + const fresh = await shape(database) + await database.exec("SET search_path TO migrated") + for (const sql of renderPgSchema(from)) await database.exec(sql) + for (const { table, rows } of seeds) for (const row of rows) { + const entries = Object.entries(row) + await database.exec(`INSERT INTO "${table.name}" (${entries.map(([column]) => `"${column}"`).join(", ")}) VALUES (${entries.map(([column, value]) => wireValue(column, value)).join(", ")})`) + } + await database.exec("BEGIN") + try { + for (const sql of migration) await database.exec(sql) + await database.exec("COMMIT") + } catch (error) { + await database.exec("ROLLBACK") + throw new FuzzFailure(`Postgres migration failed: ${String(error)}`, { migration, seeds }) + } + expect(await shape(database)).toEqual(fresh) + for (const { table, rows } of seeds) { + const next = after.find((next) => next.name === table.name) + if (next === undefined) continue + const expected = rows.map((row) => ({ id: row.id, ...Object.fromEntries(next.columns.map((column) => [column.name, Object.hasOwn(row, column.name) ? (row as Record)[column.name] : column.default ? defaults[column.name] : null])) })) + expect((await database.query(`SELECT * FROM "${table.name}" ORDER BY id`)).rows).toEqual(expected) + } + }) + }, { runs: 60 }), 300_000) +}) + diff --git a/tests/fuzz/model.ts b/tests/fuzz/model.ts index b04ab13..d98564f 100644 --- a/tests/fuzz/model.ts +++ b/tests/fuzz/model.ts @@ -9,7 +9,7 @@ // `Semantics` (dialects/types.ts) rather than its name, so a new dialect states // its rules instead of patching this file. // Inputs the model cannot judge raise `skip` with a reason. -import { fixtureRows } from "../core-cases" +import { coreFixture, type Fixture, type Order, type Customer } from "./fixtures" import type { Semantics } from "./dialects/types" import { skip } from "./harness" import type { Agg, ArithOp, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" @@ -17,37 +17,6 @@ import type { Agg, ArithOp, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" type V = number | string | null type Bool = boolean | null -interface Order { - readonly OrgId: string - readonly Id: number - readonly Customer: string - readonly Amount: number - readonly Status: string - readonly Note: string | null - /** Epoch milliseconds. */ - readonly Created: number -} -interface Customer { - readonly OrgId: string - readonly Name: string - readonly Tier: string -} - -const orders: ReadonlyArray = fixtureRows.orders.map(([OrgId, Id, Customer, Amount, Status, Note, Created]) => ({ - OrgId: OrgId as string, - Id: Id as number, - Customer: Customer as string, - Amount: Amount as number, - Status: Status as string, - Note: Note as string | null, - Created: Date.parse(`${(Created as string).replace(" ", "T")}Z`), -})) -const customers: ReadonlyArray = fixtureRows.customers.map(([OrgId, Name, Tier]) => ({ - OrgId: OrgId as string, - Name: Name as string, - Tier: Tier as string, -})) - interface Row { readonly o: Order /** The joined customer; `undefined` for an unmatched LEFT JOIN row. */ @@ -282,10 +251,10 @@ const evalCond = (env: Env, row: Row, c: Cond): Bool => { // Queries -const sourceRows = (env: Env): ReadonlyArray => - orders.flatMap((o): Array => { +const sourceRows = (env: Env, fixture: Fixture): ReadonlyArray => + fixture.orders.flatMap((o): Array => { if (env.plan.join === undefined) return [{ o }] - const matches = customers.filter((c) => c.OrgId === o.OrgId && c.Name === o.Customer) + const matches = fixture.customers.filter((c) => c.OrgId === o.OrgId && c.Name === o.Customer) if (matches.length > 0) return matches.map((c) => ({ o, c })) return env.plan.join === "left" ? [{ o }] : [] }) @@ -348,13 +317,28 @@ const ariths = (value: unknown): Array => { return [...own, ...Object.values(value).flatMap(ariths)] } -export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { +/** Constant folding is independent of relation cardinality, including an empty join. */ +const constant = (expr: Expr): boolean => { + switch (expr.k) { + case "num": + case "str": return true + case "arith": return constant(expr.l) && (typeof expr.r === "number" || constant(expr.r)) + case "coalesce": return constant(expr.e) && constant(expr.fallback) + default: return false + } +} + +export const evaluate = (plan: Plan, sem: Semantics, fixture: Fixture = coreFixture): ModelResult => { const env: Env = { sem, plan, joined: plan.join !== undefined } // A database may evaluate any expression on any row — or fold a constant one at plan time — before // filtering decides it was not needed. So a division by zero anywhere is a division by zero. const all = ariths(plan) - for (const row of sourceRows(env)) for (const e of all) evalExpr(env, row, e) - const filtered = sourceRows(env).filter((row) => and3(plan.where.map((c) => evalCond(env, row, c))) === true) + // A constant expression never reads this row. Reuse a real fixture row so the evaluator + // can check folded subexpressions even when the input fixture or its inner join is empty. + const constantRow: Row = { o: coreFixture.orders[0]! } + for (const e of all) if (constant(e)) evalExpr(env, constantRow, e) + for (const row of sourceRows(env, fixture)) for (const e of all) evalExpr(env, row, e) + const filtered = sourceRows(env, fixture).filter((row) => and3(plan.where.map((c) => evalCond(env, row, c))) === true) const orgs = new Set(filtered.flatMap((row) => [row.o.OrgId, ...(row.c ? [row.c.OrgId] : [])])) const shape = plan.shape @@ -388,7 +372,7 @@ export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { })) } - const uniqueKeys = shape.k === "rows" ? ["id"] : shape.keys.map(([alias]) => alias) + const uniqueKeys = shape.k === "rows" ? (plan.join === undefined ? ["id"] : shape.select.map(([alias]) => alias)) : shape.keys.map(([alias]) => alias) const total = uniqueKeys.length > 0 && uniqueKeys.every((key) => shape.orderBy.some(([alias]) => alias === key)) sortRows(env, rows, shape.orderBy) return { rows: page(rows, shape), ordered: total && !plan.wrap, orgs } @@ -399,8 +383,8 @@ export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { * concatenated, then the union's ORDER BY, LIMIT and OFFSET. Rows compare as * a list only when the union orders by every alias. */ -export const evaluateUnion = (plan: UnionPlan, sem: Semantics): ModelResult => { - const parts = plan.branches.map((branch) => evaluate(branch, sem)) +export const evaluateUnion = (plan: UnionPlan, sem: Semantics, fixture: Fixture = coreFixture): ModelResult => { + const parts = plan.branches.map((branch) => evaluate(branch, sem, fixture)) const rows = parts.flatMap((part) => [...part.rows]) const first = plan.branches[0]! const aliases = first.shape.k === "rows" ? first.shape.select.map(([alias]) => alias) : [] diff --git a/tests/fuzz/plan.ts b/tests/fuzz/plan.ts index e1ba22a..4475f1b 100644 --- a/tests/fuzz/plan.ts +++ b/tests/fuzz/plan.ts @@ -109,6 +109,8 @@ export interface GenOptions { readonly strings: fc.Arbitrary /** Whether the dialect has ILIKE (`FuzzDialect.ilike`). */ readonly ilike: boolean + /** Maximum recursive expression/condition depth; smoke plans default to two. */ + readonly depth?: number } /** Mostly the fixture's own numbers (Id 1–5, Amount 5/7/10/20/99), so comparisons split the rows. */ @@ -297,11 +299,11 @@ const ordering = (aliases: ReadonlyArray, unique: ReadonlyArray) const rowsShape = (joined: boolean, options: GenOptions): fc.Arbitrary => fc - .array(fc.constantFrom("num", "str", "nstr", "time").chain((sort) => expr(sort, joined, 2, options)), { minLength: 0, maxLength: 4 }) + .array(fc.constantFrom("num", "str", "nstr", "time").chain((sort) => expr(sort, joined, options.depth ?? 2, options)), { minLength: 0, maxLength: 4 }) .chain((exprs) => { - // `id` is always selected: the unique key that makes an ordering total. + // `id` identifies orders; joined rows also order by every selected alias to break duplicate-key ties. const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] - return fc.record({ distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + return fc.record({ distinct: fc.boolean(), order: ordering(select.map(([a]) => a), joined ? select.map(([a]) => a) : ["id"]) }).map( ({ distinct, order }): RowsShape => ({ k: "rows", select, distinct, ...order }), ) }) @@ -346,7 +348,7 @@ const joinArb = fc.option(fc.constantFrom("inner" as const, "left" as const), { const whereArb = (joined: boolean, options: GenOptions): fc.Arbitrary> => fc.oneof( fc.constant([]), - fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), + fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, options.depth ?? 2, options) }), { minLength: 1, maxLength: 2 }), ) /** @@ -367,9 +369,9 @@ export interface UnionPlan { const branch = (orgId: string, sorts: ReadonlyArray, options: GenOptions): fc.Arbitrary => joinArb.chain((join) => { const joined = join !== undefined - return fc.tuple(...sorts.map((sort) => expr(sort, joined, 2, options))).chain((exprs) => { + return fc.tuple(...sorts.map((sort) => expr(sort, joined, options.depth ?? 2, options))).chain((exprs) => { const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] - return fc.record({ where: whereArb(joined, options), distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + return fc.record({ where: whereArb(joined, options), distinct: fc.boolean(), order: ordering(select.map(([a]) => a), joined ? select.map(([a]) => a) : ["id"]) }).map( ({ where, distinct, order }): Plan => ({ orgId, join, where, shape: { k: "rows", select, distinct, ...order }, wrap: false, crossTenant: false }), ) }) diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index 890cc31..780d171 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -10,14 +10,16 @@ // // Every target in the registry (dialects/index.ts) runs; one whose database is // not reachable here is skipped, not dropped, so the run says so. +import fc from "fast-check" import { afterAll, describe, expect, it } from "vitest" import { compilePlan, compileUnionPlan, scopeContext, type Compiled } from "./build" import { closeAll, targets, type FuzzDialect, type FuzzTarget } from "./dialects" -import { FuzzFailure, property, skipKnown, withContext } from "./harness" +import { FuzzFailure, property, Skip, skipKnown, withContext } from "./harness" import { knownPlanIssue, knownUnionIssue } from "./known-issues" import { evaluate, evaluateUnion, type ModelResult } from "./model" import { nastyOptions, plainOptions, plan, unionPlan, type Plan, type UnionPlan } from "./plan" import { normalize } from "./values" +import { fixture, fixtureContext, type Fixture } from "./fixtures" afterAll(closeAll) @@ -30,13 +32,13 @@ const canonical = (rows: ReadonlyArray): Array => ) const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort() -const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { +const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan, data?: Fixture) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input), input.where) + return checkRows(target, input.orgId, evaluate(input, target.semantics, data), compilePlan(dialect, data === undefined ? dialect.context : fixtureContext(dialect, data), input), compilePlan(dialect, scopeContext(dialect), input), input.where) } -const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { +const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan, data?: Fixture) => { const knownUnion = knownUnionIssue({ target: target.name, plan: input }) if (knownUnion !== undefined) skipKnown(knownUnion) for (const branch of input.branches) { @@ -46,8 +48,8 @@ const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Unio return checkRows( target, input.orgId, - evaluateUnion(input, target.semantics), - compileUnionPlan(dialect, dialect.context, input), + evaluateUnion(input, target.semantics, data), + compileUnionPlan(dialect, data === undefined ? dialect.context : fixtureContext(dialect, data), input), compileUnionPlan(dialect, scopeContext(dialect), input), input.branches.map((branch) => branch.where), ) @@ -58,7 +60,8 @@ const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResul if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. - const tenantScope = scoped._tag === "Compiled" ? scoped.compiled.tenantScope : undefined + if (scoped._tag !== "Compiled") throw new FuzzFailure(`a valid plan over tenant tables was not compiled (${scoped._tag})`, { error: scoped.error }) + const tenantScope = scoped.compiled.tenantScope const outcome = await target.run(compiled) await withContext({ sql: compiled.sql, parameters: compiled.parameters, tenantScope, outcome, expected: expected.rows }, () => { expect(outcome._tag).toBe("Rows") @@ -100,3 +103,39 @@ describe("unions match the model", () => { } } }) + +// Keep the original fixed-data properties and pinned inputs; these inputs shrink data and plans together. +describe("generated fixture queries", () => { + for (const { dialect, target } of targets) { + for (const [label, data, join] of [ + ["empty orders", { orders: [], customers: [] }, undefined], + ["empty inner join", { orders: [{ OrgId: "org_1", Id: 1, Customer: "acme", Amount: 0, Status: "paid", Note: null, Created: Date.UTC(2026, 0, 1) }], customers: [] }, "inner"], + ] as const) { + it(`constant division is excluded over ${label} (${target.name})`, () => { + const input: Plan = { + orgId: "org_1", join, where: [], wrap: false, crossTenant: false, + shape: { k: "rows", select: [["id", { k: "col", col: "Id" }], ["bad", { k: "arith", op: "div", l: { k: "num", v: 1 }, r: 0 }]], distinct: false, orderBy: [] }, + } + expect(() => evaluate(input, target.semantics, data)).toThrow(Skip) + expect(() => evaluate(input, target.semantics, data)).toThrow("division by zero") + }) + } + it.skipIf(!target.available)(`empty generated tables (${target.name})`, () => matchesModel(dialect, target, { + orgId: "org_1", where: [], shape: { k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }, wrap: false, crossTenant: false, + }, { orders: [], customers: [] })) + it.skipIf(!target.available)(`duplicate generated join keys page deterministically (${target.name})`, () => matchesModel(dialect, target, { + orgId: "org_1", join: "left", where: [], shape: { k: "rows", select: [["id", { k: "col", col: "Id" }], ["tier", { k: "jcol", col: "Tier" }]], distinct: false, + orderBy: [["id", "asc"], ["tier", "asc"]], offset: 1, limit: 2 }, wrap: false, crossTenant: false, + }, { + orders: [{ OrgId: "org_1", Id: 1, Customer: "acme", Amount: 0, Status: "paid", Note: null, Created: Date.UTC(2026, 0, 1) }], + customers: ["silver", "bronze", "gold"].map((Tier) => ({ OrgId: "org_1", Name: "acme", Tier })), + })) + const options = { ...plainOptions(dialect), depth: 3 } + const plansName = `generated fixtures: query results agree (${target.name})` + it.skipIf(!target.available)(plansName, () => property(plansName, fc.record({ fixture, plan: plan(options) }), + (input) => matchesModel(dialect, target, input.plan, input.fixture), { runs: 100 }), 300_000) + const unionsName = `generated fixtures: union results agree (${target.name})` + it.skipIf(!target.available)(unionsName, () => property(unionsName, fc.record({ fixture, plan: unionPlan(options) }), + (input) => unionMatchesModel(dialect, target, input.plan, input.fixture), { runs: 60 }), 300_000) + } +}) diff --git a/tests/fuzz/registry.fuzz.test.ts b/tests/fuzz/registry.fuzz.test.ts index df6b02d..e605d83 100644 --- a/tests/fuzz/registry.fuzz.test.ts +++ b/tests/fuzz/registry.fuzz.test.ts @@ -7,6 +7,13 @@ import { readdirSync, readFileSync } from "node:fs" import { join } from "node:path" import { expect, it } from "vitest" import { dialects } from "./dialects" +import * as ClickHouseTypes from "../../src/ch/types" +import * as PostgresTypes from "../../src/pg/types" +import * as SqliteTypes from "../../src/sqlite/types" +import * as ClickHouse from "@maple-dev/effect-orm/clickhouse" +import * as Postgres from "@maple-dev/effect-orm/postgres" +import * as Sqlite from "@maple-dev/effect-orm/sqlite" +import { literalCoverage, literalExemptions } from "./literal-inventory" const sourceFiles = (dir: string): Array => readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { @@ -22,3 +29,25 @@ it("every dialect in src/ is registered with the fuzzer", () => { expect(defined.length).toBeGreaterThan(0) expect(new Set(dialects.map((d) => d.name))).toEqual(new Set(defined)) }) + +// Inspect the type modules themselves, so a new descriptor cannot be forgotten +// merely because the existing catalog still passes its own examples. +for (const [name, types, entry] of [ + ["clickhouse", ClickHouseTypes, ClickHouse], + ["postgres", PostgresTypes, Postgres], + ["sqlite", SqliteTypes, Sqlite], +] as const) { + it(`every ${name} type export has a literal property or a reason`, () => { + const covered = literalCoverage[name] + const exempt = literalExemptions[name] + const publicNames = Object.keys(types).filter((key) => key in entry) + expect([...Object.keys(covered), ...Object.keys(exempt)].sort()).toEqual(publicNames.sort()) + expect(Object.keys(covered).filter((key) => key in exempt)).toEqual([]) + for (const reason of Object.values(exempt)) expect(reason.trim().length).toBeGreaterThan(0) + for (const dialect of dialects.filter((d) => d.name === name || (name === "sqlite" && d.name === "d1"))) { + const ids = dialect.literals.map((literal) => literal.id) + expect(new Set(ids).size).toBe(ids.length) + for (const id of Object.values(covered)) expect(ids).toContain(`${dialect.name}:${id}`) + } + }) +} diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index c55aef9..430af6b 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -72,15 +72,34 @@ export const pinned: Record> = { }, ] satisfies ReadonlyArray, + // ClickHouse 26.2 must page a DISTINCT query over generated rows correctly. + "generated fixtures: query results agree": [{ + fixture: { + orders: [1, 2].map((Id) => ({ OrgId: "org_1", Id, Customer: "acme", Amount: 0, Status: "paid", Note: null, Created: Date.UTC(2026, 0, 1) })), + customers: [], + }, + plan: rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: true, orderBy: [["id", "asc"]], offset: 1 }), + }], + // Literals. "clickhouse:Float64": [5e-324, -5e-324, 44812297868494010, 2.1722934738885276e269], "clickhouse:Array(Nullable(Int64)) compare": [[0, 4294967296]], "clickhouse:String": ["__PARAM_string_x__", "\\'", "\0"], "postgres:boolean select": [false], + "postgres:float4 ": [Math.fround(0.1), 2 ** -149, -(2 ** -149), Math.fround(3.4028234663852886e38)], // SQLite reads these decimals as a neighbouring double, as a literal or through json_extract. ":real ": [1.23456789e-95, 5.464465248610941e-277, -1.415281131826549e-267, 0, 5e-324], "postgres:text[] select": [[]], - "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + "postgres:timestamptz ": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + + // Timestamp years 0 and 99 must survive without Date.UTC's two-digit-year coercion. + "sqlite:timestamp ": [DateTime.makeUnsafe("0000-01-01T00:00:00.000Z"), DateTime.makeUnsafe("0099-12-31T23:59:59.999Z")], + "d1:timestamp ": [DateTime.makeUnsafe("0000-01-01T00:00:00.000Z"), DateTime.makeUnsafe("0099-12-31T23:59:59.999Z")], + + // Schema-derived JSON keeps explicit SQL escaping and nested-value regressions. + "postgres:jsonb ": [{ message: "__PARAM_string_x__\\'", values: [null, -0, 5e-324], meta: { active: false, tags: ["it's", "😀"] } }], + "sqlite:json ": [{ message: "__PARAM_string_x__\\'\0", values: [null, 1], meta: { active: true, tags: ["it's", "😀"] } }], + "d1:json ": [{ message: "__PARAM_string_x__\\'\0", values: [null, 1], meta: { active: true, tags: ["it's", "😀"] } }], // SQLite migrations. "migrations reach": [ diff --git a/tests/fuzz/values.ts b/tests/fuzz/values.ts index b3c559c..0b8c2af 100644 --- a/tests/fuzz/values.ts +++ b/tests/fuzz/values.ts @@ -2,7 +2,7 @@ // (`dialects/*.ts`): strings that break SQL writers, times, and normalising // decoded values for comparison. import fc from "fast-check" -import { DateTime } from "effect" +import { Arbitrary, DateTime, Schema } from "effect" import type * as CH from "@maple-dev/effect-orm/clickhouse" import type { StringRules } from "./dialects/types" @@ -111,12 +111,14 @@ export interface LiteralType { /** `:`; appears in property names, `regressions.ts` and `known-issues.ts`. */ readonly id: string readonly type: CH.CHType - readonly value: fc.Arbitrary + readonly value: fc.Arbitrary | Arbitrary.Arbitrary /** The value as the JSON the database parses. */ readonly wire: (value: unknown) => unknown readonly source: (json: string, value: unknown) => string /** Whether `$.v.eq(value)` is meaningful for this type. */ readonly comparable: boolean + /** Canonicalize a driver representation with a coarser textual precision (e.g. float4). */ + readonly normalizeValue?: (value: unknown) => unknown } export const finite = fc.double({ noNaN: true, noDefaultInfinity: true }) @@ -124,3 +126,14 @@ export const int32 = fc.integer({ min: -(2 ** 31), max: 2 ** 31 - 1 }) export const nullOr = (arb: fc.Arbitrary) => fc.option(arb, { nil: null, freq: 4 }) export const dateTimeWire = (fraction: boolean) => (value: unknown) => wireDateTime(DateTime.toEpochMillis(value as DateTime.Utc), fraction) + +/** One schema supplies nested JSON payloads and their native shrinking. */ +export const jsonPayloadSchema = Schema.Struct({ + message: Schema.String.check(Schema.isMaxLength(40)), + values: Schema.Array(Schema.NullOr(Schema.Finite)).check(Schema.isMaxLength(4)), + meta: Schema.Struct({ active: Schema.Boolean, tags: Schema.Array(Schema.String.check(Schema.isMaxLength(16))).check(Schema.isMaxLength(3)) }), +}) +export const jsonPayload = (rules: StringRules) => Arbitrary.schema(jsonPayloadSchema).pipe( + Arbitrary.filter((value) => wellFormed(value) && [value.message, ...value.meta.tags].every((s) => + (rules.nul || !s.includes("\0")) && (rules.leadingBom || !s.startsWith("\ufeff")))), +) From 5cde8de642c4354292cd7c28b545d525fbf210f3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 26 Oct 2026 19:19:33 +0100 Subject: [PATCH 2/2] test: clarify native discards and compile Postgres codec cases --- tests/fuzz/README.md | 6 ++++++ tests/fuzz/codecs.fuzz.test.ts | 2 +- tests/fuzz/harness.test.ts | 19 +++++++++++++++++++ tests/fuzz/harness.ts | 11 +++++++++-- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index e8197c3..ee540f3 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -35,6 +35,12 @@ checks compositional encode/decode behavior. The shared harness accepts native Effect arbitraries directly, preserving shrinking and replay; do not sample them into fast-check constants, which loses their shrink tree. +Native properties discard inputs during generation with `Arbitrary.filter` (or +schema checks). Effect's checker has no predicate-level discard result: throwing +`skip()` or `skipKnown()` inside a native predicate fails with a diagnostic. Keep +oracle limitations in native generator filters; rejected candidates do not consume +the run budget, and exhausting generation fails instead of passing with no checks. + Keep fast-check's constrained generators for SQL domains, adversarial strings and well-typed query/migration plans. A descriptor's wire codec can be broader than its storage type: `UInt8.schema` accepts finite numbers, while valid SQL inputs diff --git a/tests/fuzz/codecs.fuzz.test.ts b/tests/fuzz/codecs.fuzz.test.ts index e44e8c6..fa9ccc9 100644 --- a/tests/fuzz/codecs.fuzz.test.ts +++ b/tests/fuzz/codecs.fuzz.test.ts @@ -94,7 +94,7 @@ describe("Effect generated compiled row codecs", () => { describe("Effect generated numeric wire representations", () => { for (const [name, type] of [["clickhouse", CH.float64], ["postgres", PG.float8]] as const) { const table = CH.table("numeric_rows", { external: true, columns: { value: type } }) - const compiled = CH.compileUnsafe(CH.from(table).select()) + const compiled = (name === "postgres" ? PG.compileUnsafe : CH.compileUnsafe)(CH.from(table).select()) const label = `quoted and unquoted numeric wire agree (${name})` it(label, () => property(label, Arbitrary.schema(Schema.Finite), async (value) => { const decoded = await Effect.runPromise(compiled.decodeRows([{ value }, { value: String(value) }])) diff --git a/tests/fuzz/harness.test.ts b/tests/fuzz/harness.test.ts index ec6c872..299f9a2 100644 --- a/tests/fuzz/harness.test.ts +++ b/tests/fuzz/harness.test.ts @@ -32,6 +32,25 @@ describe("fuzz harness coverage guarantees", () => { await new Promise((resolve) => setTimeout(resolve, 10)) }, { runs: 100, timeoutMs: 1 })).rejects.toThrow("interrupted budget") }) + it("native filters discard inputs without consuming the checked budget", async () => { + reset() + let checked = 0 + const arb = Arbitrary.schema(Schema.Literals([0, 1])).pipe(Arbitrary.filter((n) => n === 1)) + await property("filtered native", arb, (n) => { expect(n).toBe(1); checked++ }, { runs: 5 }) + expect(checked).toBe(5) + }) + it("fails when a native generator discards every input", async () => { + reset() + let checked = 0 + await expect(property("all native discards", Arbitrary.Constant(1).pipe(Arbitrary.filter(() => false)), () => { checked++ }, { runs: 1 })).rejects.toThrow("Property exhausted after 0 run(s)") + expect(checked).toBe(0) + }) + it("explains that predicate-level skips need native generator filtering", async () => { + reset() + for (const discard of [skip, skipKnown]) { + await expect(property("native predicate skip", Arbitrary.Constant(1), () => discard("outside oracle"), { runs: 1 })).rejects.toThrow("Native properties must use Arbitrary.filter") + } + }) it("preserves native shrinking, failure context and replay", async () => { reset() const arb = Arbitrary.schema(Schema.Int) diff --git a/tests/fuzz/harness.ts b/tests/fuzz/harness.ts index 0d132bf..e1305ac 100644 --- a/tests/fuzz/harness.ts +++ b/tests/fuzz/harness.ts @@ -181,10 +181,17 @@ const nativeProperty = async ( options: PropertyOptions, ): Promise => { if (config.path !== undefined) throw new Error("FUZZ_PATH is for fast-check properties; select one with -t") + const evaluate = async (input: T): Promise => { + try { await predicate(input) } catch (error) { + // Effect only discards during generation; a property failure is always falsified. + if (error instanceof Skip) throw new FuzzFailure("Native properties must use Arbitrary.filter to discard inputs; skip()/skipKnown() cannot discard after evaluation", { reason: error.reason }) + throw error + } + } // Pinned examples are independent of the generator and must keep working after changes to it. - for (const input of pinnedFor(name)) await predicate(input as T) + for (const input of pinnedFor(name)) await evaluate(input as T) const check = Arbitrary.checkEffect(arb, (input) => Effect.tryPromise({ - try: async () => { await predicate(input); return true }, + try: async () => { await evaluate(input); return true }, catch: (error) => error, }), effectOptions(options.runs)) const result = await Effect.runPromise(options.timeoutMs === undefined ? check : check.pipe(Effect.timeout(options.timeoutMs)))