test(fuzz): one registry of dialects, so every suite fuzzes every dialect

.github/workflows/fuzz.yml

@@ -0,0 +1,55 @@
+name: Fuzz
+
+# A soak of every fuzz property on every target. `bun run test` runs the same
+# properties at their smoke budget on each PR; this runs them at 20×, nightly,
+# so a rare counterexample is found here and not by a user. A failure prints
+# the replay command and the input to pin (tests/fuzz/README.md).
+on:
+    schedule:
+        - cron: "17 3 * * *"
+    workflow_dispatch:
+        inputs:
+            scale:
+                description: "FUZZ_SCALE: multiplies each property's runs"
+                default: "20"
+
+permissions:
+    contents: read
+
+jobs:
+    soak:
+        name: Soak / ClickHouse ${{ matrix.clickhouse }}
+        runs-on: ubuntu-latest
+        timeout-minutes: 60
+        strategy:
+            fail-fast: false
+            matrix:
+                clickhouse: ["26.2.19.43", "26.8.2.7"]
+        services:
+            clickhouse:
+                image: clickhouse/clickhouse-server:${{ matrix.clickhouse }}
+                env:
+                    CLICKHOUSE_USER: maple
+                    CLICKHOUSE_PASSWORD: maple
+                    CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+                ports:
+                    - 8123:8123
+                options: >-
+                    --health-cmd "clickhouse-client --user maple --password maple --query 'SELECT 1'"
+                    --health-interval 5s
+                    --health-timeout 5s
+                    --health-retries 20
+        steps:
+            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+              with:
+                  persist-credentials: false
+            - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
+            - run: bun install --frozen-lockfile
+            - name: Fuzz every target
+              env:
+                  EFFECT_ORM_CLICKHOUSE_URL: http://127.0.0.1:8123
+                  EFFECT_ORM_CLICKHOUSE_USER: maple
+                  EFFECT_ORM_CLICKHOUSE_PASSWORD: maple
+                  FUZZ_SCALE: ${{ inputs.scale || '20' }}
+                  FUZZ_VERBOSE: "1"
+              run: bun run fuzz

tests/fuzz/README.md

@@ -0,0 +1,82 @@
+# The fuzzer
+
+Property-based tests for the query builder, run on real databases. They catch
+what hand-written cases miss: SQL that runs but returns the wrong rows, escaping
+bugs, and values the builder throws on instead of refusing.
+
+```sh
+bun run test                 # every property at its smoke budget (part of the normal suite)
+bun run fuzz                 # the fuzz suites only
+FUZZ_SCALE=20 bun run fuzz   # a soak: 20× each property's runs (the nightly workflow)
+```
+
+ClickHouse targets run when `EFFECT_ORM_CLICKHOUSE_URL` (and `_USER`, `_PASSWORD`) are set
+and are skipped otherwise. Postgres runs in-process on PGlite.
+
+## Suites
+
+| File | Property | Oracle |
+| --- | --- | --- |
+| `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target |
+| `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser |
+| `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants |
+| `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — |
+
+## Adding a dialect
+
+Write `dialects/<name>.ts` exporting a `FuzzDialect` (see `dialects/types.ts`)
+and add it to `dialects/index.ts`. `registry.fuzz.test.ts` fails until you do.
+Every suite then runs on it with no other edit.
+
+A `FuzzDialect` gives:
+
+- `module`: the package entry (`/postgres`, …) the plans are built and compiled with.
+- `context`: the core fixture as CTEs over literal rows (`tests/core-cases.ts`),
+  so every database reads the same rows.
+- `strings` and `ilike`: what the generators may produce for it.
+- `literals`: its column types, each with a way to build the reference row
+  *without* the builder (usually the database parsing JSON).
+- `targets`: the databases it runs on. Each target states its `Semantics`, the
+  points where databases legitimately disagree (NULL ordering, integer division,
+  what an unmatched LEFT JOIN column holds, LIKE case rules, …). The model reads
+  only these. **Never branch on a target's name in `model.ts`.** If a database
+  differs in a new way, add a field to `Semantics` and set it for every target.
+
+A difference belongs in `Semantics` only when it is the database's documented
+behaviour and the builder cannot reasonably hide it. When the builder *could*
+make the dialects agree, the difference is a bug: record it in
+`known-issues.ts` and fix it.
+
+## Writing generators that find bugs
+
+- **Derive values from the fixture.** A random string almost never equals a
+  fixture value, so `eq` and `LIKE` over random strings test only the empty
+  result. `plan.ts` mixes in fixture values (`FIXTURE_STRINGS`), the fixture's
+  numbers (`smallInt`), times around `Created`, and LIKE patterns made by
+  editing fixture values (re-cased, wildcarded, escaped). That last change is
+  what made SQLite's case-insensitive, escape-less LIKE visible.
+- **Keep plans as data.** The generator never calls the builder: `plan.ts` makes
+  a `Plan`, `build.ts` turns it into builder calls, `model.ts` evaluates it. A
+  failure prints the shrunk plan, and the plan can be pinned.
+- **Well-typed by construction.** Expressions are generated by sort, so a
+  refusal from the builder is a finding, not noise.
+- **Check that inputs are judged.** Run with `FUZZ_VERBOSE=1`: each property
+  prints how many inputs it checked and why it skipped the rest. A property that
+  skips most of its inputs is testing nothing.
+
+## When a property fails
+
+The report prints the shrunk input, the SQL, the outcome, and a replay command:
+
+```sh
+FUZZ_SEED=… FUZZ_PATH=… bun run fuzz -t 'plans match the model \(postgres, plain strings\)'
+```
+
+1. Decide whose bug it is: the builder, the model, or the database.
+2. Builder bug: fix it, then pin the input in `regressions.ts` so it is replayed
+   before any random input from then on.
+3. Not fixable now: add an entry to `known-issues.ts` with a reason someone can act
+   on. The suite stays green and the rest of the input space keeps being fuzzed.
+   `FUZZ_STRICT=1` (or `FUZZ_STRICT=<id>`) turns entries back on to check a fix.
+4. Model bug or a legitimate database difference: fix `model.ts`, or add a
+   `Semantics` field (see above).

tests/fuzz/build.ts

@@ -1,13 +1,13 @@
-// A plan as builder calls, for either dialect.
+// A plan as builder calls, for any dialect.
 //
 // The builder is driven through its public, typed surface exactly as an
 // application would; `any` here only erases the static types a random plan
 // cannot have. Params are named up front from the plan's structure so that
 // compiling the same plan twice resolves the same names to the same values.
 import { DateTime, Effect, Exit } from "effect"
-import * as CH from "@maple-dev/effect-orm/clickhouse"
-import * as PG from "@maple-dev/effect-orm/postgres"
+import type * as CH from "@maple-dev/effect-orm/clickhouse"
 import type { CoreContext } from "../core-cases"
+import type { FuzzDialect } from "./dialects/types"
 import type { Agg, Cond, Expr, Operand, Plan } from "./plan"
 
 const pad = (n: number, width = 2) => String(n).padStart(width, "0")
@@ -74,7 +74,8 @@ export interface Built {
 	readonly params: Record<string, unknown>
 }
 
-export const buildPlan = (ctx: CoreContext, plan: Plan): Built => {
+export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Built => {
+	const { lit, param, and, or, not, rawCond, fromQuery } = dialect.module
 	const { names, params } = nameParams(plan)
 
 	const expr = ($: any, e: Expr): any => {
@@ -85,7 +86,7 @@ export const buildPlan = (ctx: CoreContext, plan: Plan): Built => {
 				return $.c[e.col]
 			case "num":
 			case "str":
-				return CH.lit(e.v as never)
+				return lit(e.v as never)
 			case "arith":
 				return expr($, e.l)[e.op](typeof e.r === "number" ? e.r : expr($, e.r))
 			case "coalesce":
@@ -99,7 +100,7 @@ export const buildPlan = (ctx: CoreContext, plan: Plan): Built => {
 			case "time":
 				return op.form === "DateTime" ? DateTime.makeUnsafe(op.ms) : op.form === "Date" ? new Date(op.ms) : timeString(op.ms)
 			case "param":
-				return (CH.param as any)[op.kind](names.get(op)!)
+				return (param as any)[op.kind](names.get(op)!)
 			case "expr":
 				return expr($, op.e)
 		}
@@ -119,15 +120,15 @@ export const buildPlan = (ctx: CoreContext, plan: Plan): Built => {
 			case "and":
 			case "or": {
 				const parts = c.cs.map((x) => cond($, x))
-				if (c.style === "fn") return (c.k === "and" ? CH.and : CH.or)(...parts)
+				if (c.style === "fn") return (c.k === "and" ? and : or)(...parts)
 				return parts.slice(1).reduce((acc, part) => acc[c.k](part), parts[0])
 			}
 			case "not":
-				return CH.not(cond($, c.c))
+				return not(cond($, c.c))
 			case "const":
-				return CH.rawCond(c.v ? "1 = 1" : "1 = 0")
+				return rawCond(c.v ? "1 = 1" : "1 = 0")
 			case "tenant":
-				return (c.side === "orders" ? $ : $.c).OrgId.eq(CH.param.string("orgId"))
+				return (c.side === "orders" ? $ : $.c).OrgId.eq(param.string("orgId"))
 		}
 	}
 	const agg = ($: any, a: Agg): any => {
@@ -166,14 +167,24 @@ export const buildPlan = (ctx: CoreContext, plan: Plan): Built => {
 	if (shape.limit !== undefined) q = q.limit(shape.limit)
 	if (shape.offset !== undefined) q = q.offset(shape.offset)
 	if (plan.crossTenant) q = q.crossTenant()
-	if (plan.wrap === "all") q = CH.fromQuery(q, "w").select()
+	if (plan.wrap === "all") q = fromQuery(q, "w").select()
 	if (plan.wrap === "names") {
 		const names = shape.k === "rows" ? shape.select.map(([alias]) => alias) : [...shape.keys, ...shape.aggs].map(([alias]) => alias)
-		q = CH.fromQuery(q, "w").select(...names)
+		q = fromQuery(q, "w").select(...names)
 	}
 	return { query: q, params }
 }
 
+/**
+ * The dialect's context over the plain tables, with no fixture CTEs.
+ *
+ * The fixture CTEs are bare SQL the builder cannot see into, and a CTE named
+ * `orders` shadows the `orders` table, so a query compiled against them never
+ * reports a meaningful tenant scope. Compiled here, the scope the builder
+ * reports is the one a real application gets.
+ */
+export const scopeContext = (dialect: FuzzDialect): CoreContext => ({ ...dialect.context, from: (table) => dialect.module.from(table) as any })
+
 export type Compiled =
 	| { readonly _tag: "Compiled"; readonly compiled: CH.CompiledQuery<any> }
 	| { readonly _tag: "Refused"; readonly error: string }
@@ -185,11 +196,10 @@ export type Compiled =
  * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that
  * escapes `compile` — or the builder calls before it — is `Threw`.
  */
-export const compilePlan = (ctx: CoreContext, plan: Plan): Compiled => {
+export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => {
 	try {
-		const { query, params } = buildPlan(ctx, plan)
-		const compile = ctx.dialect === "clickhouse" ? CH.compile : PG.compile
-		const exit = Effect.runSyncExit((compile as any)(query, params) as Effect.Effect<CH.CompiledQuery<any>, unknown>)
+		const { query, params } = buildPlan(dialect, ctx, plan)
+		const exit = Effect.runSyncExit(dialect.module.compile(query, params))
 		if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value }
 		const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail")
 		return failure !== undefined

tests/fuzz/compile.fuzz.test.ts

@@ -3,19 +3,17 @@
 //   plans compile cleanly   a well-typed plan is never refused, and nothing throws
 //   deterministic           compiling the same plan twice gives the same SQL and params
 //   no leaked JS            no `undefined`, `[object …]`, `NaN` or `__PARAM_` in the SQL
-//   placeholders line up    Postgres `$n` are exactly 1..parameters.length
+//   placeholders line up    the dialect's own check, e.g. Postgres `$n` are exactly 1..parameters.length
 //   tenant scope            `.crossTenant()` is always cross-tenant; a top-level tenant
 //                           predicate on an unjoined query is always single-tenant
 //   hostile values          any JS value where a value goes is compiled or refused with a
 //                           typed QueryBuilderError — never a throw, never a defect
 import { describe, expect, it } from "vitest"
 import fc from "fast-check"
 import { Effect, Exit } from "effect"
-import * as CH from "@maple-dev/effect-orm/clickhouse"
-import * as PG from "@maple-dev/effect-orm/postgres"
 import type { CoreContext } from "../core-cases"
-import { scopeContexts as contexts } from "./contexts"
-import { compilePlan } from "./build"
+import { compilePlan, scopeContext } from "./build"
+import { dialects, type FuzzDialect } from "./dialects"
 import { FuzzFailure, property, skipKnown } from "./harness"
 import { knownHostileIssue, knownPlanIssue } from "./known-issues"
 import { nastyOptions, ORDER_COLUMNS, plan, type OrderColumn, type Plan } from "./plan"
@@ -25,18 +23,19 @@ const stringsIn = (value: unknown): Array<string> =>
 
 const serialize = (value: unknown) => JSON.stringify(value, (_, v) => (typeof v === "bigint" ? `${v}n` : v instanceof Date ? `Date(${v.getTime()})` : v))
 
-const compiles = (ctx: CoreContext, input: Plan) => {
-	const known = knownPlanIssue({ target: ctx.dialect, plan: input })
+const compiles = (dialect: FuzzDialect, input: Plan) => {
+	const known = knownPlanIssue({ target: dialect.name, plan: input })
 	if (known !== undefined) skipKnown(known)
-	const first = compilePlan(ctx, input)
+	const ctx = scopeContext(dialect)
+	const first = compilePlan(dialect, ctx, input)
 	if (first._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${first._tag})`, { error: first.error })
 	const { compiled } = first
 	const context = { sql: compiled.sql, parameters: serialize(compiled.parameters), tenantScope: compiled.tenantScope }
 	const fail = (message: string): never => {
 		throw new FuzzFailure(message, context)
 	}
 
-	const second = compilePlan(ctx, input)
+	const second = compilePlan(dialect, ctx, input)
 	if (second._tag !== "Compiled" || second.compiled.sql !== compiled.sql || serialize(second.compiled.parameters) !== serialize(compiled.parameters)) {
 		fail("compiling the same plan twice gave different output")
 	}
@@ -46,12 +45,10 @@ const compiles = (ctx: CoreContext, input: Plan) => {
 		if (compiled.sql.includes(token) && !strings.some((s) => s.includes(token))) fail(`the SQL contains ${JSON.stringify(token)}`)
 	}
 
-	if (ctx.dialect === "postgres" && !strings.some((s) => s.includes("$"))) {
-		const used = new Set([...compiled.sql.matchAll(/\$(\d+)/g)].map((m) => Number(m[1])))
-		const expected = Array.from({ length: compiled.parameters.length }, (_, i) => i + 1)
-		if (used.size !== expected.length || !expected.every((n) => used.has(n))) {
-			fail(`placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${compiled.parameters.length} parameters`)
-		}
+	// A plan's own strings may contain placeholder syntax (`$1`, `?`) that is rightly bound, not counted.
+	if (dialect.checkPlaceholders !== undefined && !strings.some((s) => /[$?:{]/.test(s))) {
+		const problem = dialect.checkPlaceholders(compiled.sql, compiled.parameters.length)
+		if (problem !== undefined) fail(problem)
 	}
 
 	if (input.crossTenant && compiled.tenantScope !== "cross-tenant") fail(".crossTenant() did not make the query cross-tenant")
@@ -62,9 +59,9 @@ const compiles = (ctx: CoreContext, input: Plan) => {
 }
 
 describe("compile properties", () => {
-	for (const ctx of Object.values(contexts)) {
-		const name = `plans compile cleanly (${ctx.dialect})`
-		it(name, () => property(name, plan(nastyOptions(ctx.dialect)), (input) => compiles(ctx, input), { runs: 300 }), 120_000)
+	for (const dialect of dialects) {
+		const name = `plans compile cleanly (${dialect.name})`
+		it(name, () => property(name, plan(nastyOptions(dialect)), (input) => compiles(dialect, input), { runs: 300 }), 120_000)
 	}
 })
 
@@ -79,23 +76,24 @@ const hostile: fc.Arbitrary<unknown> = fc.oneof(
 
 type HostileSite = "lit" | "add" | "eq" | "neq" | "gt" | "in_" | "notIn" | "between" | "like" | "param.int" | "param.float" | "param.string" | "param.bool" | "param.dateTime" | "param.of" | "select"
 export interface HostileInput {
-	readonly dialect: "clickhouse" | "postgres"
+	readonly dialect: string
 	readonly column: OrderColumn
 	readonly site: HostileSite
 	readonly value: unknown
 }
 
-const hostileQuery = (ctx: CoreContext, input: HostileInput) => {
+const hostileQuery = (dialect: FuzzDialect, ctx: CoreContext, input: HostileInput) => {
 	const { site, value, column } = input
+	const { lit, param } = dialect.module
 	const params: Record<string, unknown> = { orgId: "org_1", h: value }
 	let q: any = ctx.from(ctx.orders)
-	if (site === "select") return { query: q.select(() => ({ id: CH.lit(1), bad: value })), params }
-	if (site === "lit") return { query: q.select(() => ({ id: CH.lit(value as never) })), params }
+	if (site === "select") return { query: q.select(() => ({ id: lit(1), bad: value })), params }
+	if (site === "lit") return { query: q.select(() => ({ id: lit(value as never) })), params }
 	if (site === "add") return { query: q.select(($: any) => ({ id: $.Amount.add(value) })), params }
 	q = q.select(($: any) => ({ id: $.Id }))
 	if (site.startsWith("param.")) {
 		const kind = site.slice("param.".length)
-		const marker = kind === "of" ? CH.param.of((ctx.orders as any).columns[column], "h") : (CH.param as any)[kind]("h")
+		const marker = kind === "of" ? param.of((ctx.orders as any).columns[column], "h") : (param as any)[kind]("h")
 		return { query: q.where(($: any) => [$[column].eq(marker)]), params }
 	}
 	return {
@@ -104,21 +102,21 @@ const hostileQuery = (ctx: CoreContext, input: HostileInput) => {
 	}
 }
 
-const hostileInput = (dialect: "clickhouse" | "postgres"): fc.Arbitrary<HostileInput> =>
+const hostileInput = (dialect: string): fc.Arbitrary<HostileInput> =>
 	fc.record({
 		dialect: fc.constant(dialect),
 		column: fc.constantFrom(...(Object.keys(ORDER_COLUMNS) as OrderColumn[])),
 		site: fc.constantFrom<HostileSite>("lit", "add", "eq", "neq", "gt", "in_", "notIn", "between", "like", "param.int", "param.float", "param.string", "param.bool", "param.dateTime", "param.of", "select"),
 		value: hostile,
 	})
 
-const neverThrows = (ctx: CoreContext, input: HostileInput) => {
+const neverThrows = (dialect: FuzzDialect, input: HostileInput) => {
 	const known = knownHostileIssue(input)
 	if (known !== undefined) skipKnown(known)
 	let exit: Exit.Exit<unknown, unknown>
 	try {
-		const { query, params } = hostileQuery(ctx, input)
-		exit = Effect.runSyncExit(((ctx.dialect === "clickhouse" ? CH.compile : PG.compile) as any)(query, params))
+		const { query, params } = hostileQuery(dialect, scopeContext(dialect), input)
+		exit = Effect.runSyncExit(dialect.module.compile(query, params))
 	} catch (error) {
 		throw new FuzzFailure("the builder threw instead of failing with a QueryBuilderError", {
 			error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack?.split("\n").slice(1, 6).join("\n")}` : String(error),
@@ -138,8 +136,8 @@ const neverThrows = (ctx: CoreContext, input: HostileInput) => {
 }
 
 describe("hostile values", () => {
-	for (const ctx of Object.values(contexts)) {
-		const name = `hostile values are refused, never thrown (${ctx.dialect})`
-		it(name, () => property(name, hostileInput(ctx.dialect), (input) => neverThrows(ctx, input), { runs: 500 }), 120_000)
+	for (const dialect of dialects) {
+		const name = `hostile values are refused, never thrown (${dialect.name})`
+		it(name, () => property(name, hostileInput(dialect.name), (input) => neverThrows(dialect, input), { runs: 500 }), 120_000)
 	}
 })