diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md new file mode 100644 index 0000000..dc4ec1e --- /dev/null +++ b/.changeset/clickhouse-union-numbers.md @@ -0,0 +1,9 @@ +--- +"@maple-dev/effect-orm": patch +--- + +On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1` +and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic +are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by +`CAST(x AS Nullable(Float64)), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the +values themselves are not converted. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 8c3f5e2..4d591fc 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -28,9 +28,15 @@ themselves keep whatever ordering they declared. Unions compile with **`compileUnion`**, not `compile`. -> **Watch branch types.** ClickHouse unifies column types across `UNION ALL` branches, and a -> branch pairing (say) a `Float64` with a `UInt64` can be rejected outright depending on server -> settings. Cast explicitly in each branch when the types are not already identical. +> **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and +> `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, +> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy` +> sorts a numeric column whose branches may differ in type (different declared types, or different +> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `CAST(x AS Nullable(Float64))` and then +> `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values +> themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them +> exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that +> needs one type, should cast it in each branch. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index e889029..d67fe58 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -159,6 +159,22 @@ describe("CompiledQuery.decodeRows", () => { expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] }) }) + it("orders a ClickHouse union's mixed numeric column by exact keys, leaving its values alone", () => { + // `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses. + const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) + const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) + const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {}) + // The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact. + expect(compiled.sql).not.toContain("toFloat64(orders") + expect(compiled.sql).toContain("ORDER BY CAST(shifted AS Nullable(Float64)) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") + // Different declared types are mixed too; one declared type in every branch is not. + const mixed = compileUnionUnsafe( + CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]), + {}, + ) + expect(mixed.sql).toContain("ORDER BY CAST(n AS Nullable(Float64)) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") + }) + it.effect("decodes rows with the declared schema for handwritten SQL", () => Effect.gen(function* () { const compiled = rawCompiledQuery<{ readonly name: string; readonly count: number }>({ diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..afb784b 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -8,7 +8,7 @@ // 4. Assembling into SqlQuery and calling the existing compileQuery() import { compiled, compiledUnsafe, fail } from "./failure" -import { columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" +import { CHFloatResult, CHNumber, columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" import type { CHQuery, CHQueryState, NeedsSelect } from "./query" import type { CHUnionQuery } from "./union" import { isInsert, type AnyInsert, type CHInsert } from "./insert" @@ -22,7 +22,7 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { track, untracked, type RenderTrack } from "../sql/render-tracker" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param" -import { mergeResultSchemas } from "./define-fn" +import { mergeResultSchemas, schemaOf } from "./define-fn" import { encodeValue } from "./literal" import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" import { Effect, Option, Schema } from "effect" @@ -1391,6 +1391,39 @@ export function compileUnionRaw>( } /** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */ +/** A JS-number schema: every numeric ClickHouse type and arithmetic result decodes through one of these. */ +const isNumberSchema = (schema: unknown): boolean => { + if (schema === CHNumber || schema === CHFloatResult) return true + // NullOr(CHNumber): a nullable column or arithmetic over one. + const members = (schema as { members?: ReadonlyArray } | undefined)?.members + return Array.isArray(members) && members.some((m) => m === CHNumber || m === CHFloatResult) && members.every((m) => m === CHNumber || m === CHFloatResult || m === Schema.Null) +} + +/** + * The union columns a dialect with `unionNumberOrderKeys` orders through those + * keys: every branch selects a number, and their types may differ, because the + * branches declare different column types or compute it differently + * (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). + */ +function mixedNumberColumns(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { + const casts = new Set() + if (currentDialect().unionNumberOrderKeys === undefined || queries.length < 2) return casts + for (const alias of selectKeys) { + const exprs = queries.map((q) => (selectExprsOf(q) as Record | undefined)?.[alias]) + const kinds = exprs.map((expr) => { + if (!isExprLike(expr)) return undefined + const declared = columnTypeOf(expr as Expr) + if (declared !== undefined) return isNumberSchema(declared.schema) ? declared.sql : undefined + return isNumberSchema(schemaOf(expr)) ? "computed" : undefined + }) + if (kinds.some((kind) => kind === undefined)) continue + if (new Set(kinds).size > 1) casts.add(alias) + // The same computation in every branch (a `sum` over one column) has one type; different ones may not. + else if (kinds[0] === "computed" && new Set(exprs.map((expr) => compileSqlFragment((expr as Expr).toFragment()))).size > 1) casts.add(alias) + } + return casts +} + function compileUnionInner, Params extends Record>( union: CHUnionQuery, params: Params, @@ -1419,6 +1452,7 @@ function compileUnionInner, Params extends Re const subQueries = state.queries.map((q) => compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) + const mixedNumbers = mixedNumberColumns(state.queries, selectKeys) const bounds = new Set( subQueries.flatMap((q) => { const bound = tenantBoundOf(q) @@ -1451,7 +1485,14 @@ function compileUnionInner, Params extends Re if (hasOuter) { sql = `SELECT * FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__union")}` : ""}` if (state.outerOrderBySpecs.length > 0) { - sql += `\nORDER BY ${orderByClause(state.outerOrderBySpecs).join(", ")}` + const orderKeys = currentDialect().unionNumberOrderKeys + const clause = orderByClause(state.outerOrderBySpecs).map((rendered, i) => { + const [column, direction] = state.outerOrderBySpecs[i]! + return orderKeys !== undefined && mixedNumbers.has(column) + ? orderKeys(quoteIdent(column)).map((key) => `${key} ${direction.toUpperCase()}`).join(", ") + : rendered + }) + sql += `\nORDER BY ${clause.join(", ")}` } if (state.outerLimitValue != null) { sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b6c97..07ec134 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -151,6 +151,15 @@ export interface Dialect extends Omit { * unchecked: a custom dialect says which set it renders, if either. */ readonly functions?: string + /** + * The ORDER BY keys for a numeric UNION ALL column whose branches may have + * different types. ClickHouse has no common type for `UInt64` and `Int64` + * (or `Int64` and `Float64`): the column is a `Variant`, which ORDER BY + * refuses as a key. The values are left as they are (an INSERT ... SELECT or + * an IN over the union keeps them exact); only the sort key is converted. + * Absent: the dialect orders such a column as it is. + */ + readonly unionNumberOrderKeys?: (column: string) => ReadonlyArray } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -173,6 +182,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", + // CAST, not toFloat64: ClickHouse 26.2's toFloat64 refuses a Variant. Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. + unionNumberOrderKeys: (column) => [`CAST(${column} AS Nullable(Float64))`, `accurateCastOrNull(${column}, 'Int128')`], } // The dialect of the enclosing compile, beside the syntax installed for the diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 25a6b4f..4c34553 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -17,7 +17,8 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | File | Property | Oracle | | --- | --- | --- | -| `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target | +| `queries.fuzz.test.ts` | random plans, and `unionAll`s of them (branch and union ORDER BY / LIMIT / OFFSET), return the rows `model.ts` computes | a JS model of SQL, per target | +| `writes.fuzz.test.ts` | random sequences of inserts (multi-row, defaulted cells, INSERT … SELECT, ON CONFLICT), updates and deletes, with RETURNING, leave a scratch table as the model says; unsupported writes are refused | a JS model of the table's rows | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | @@ -36,6 +37,14 @@ A `FuzzDialect` gives: - `strings` and `ilike`: what the generators may produce for it. - `literals`: its column types, each with a way to build the reference row *without* the builder (usually the database parsing JSON). +- `writes`: which writes its builder compiles (`insertSelect`, `update`, `delete`, + `onConflict`, `returning`, `defaultCells` for a multi-row VALUES that leaves a + defaulted column out of some rows), and a `scratch` table: its query `Table`, + the DDL that creates and drops it, and `exec` to run that DDL on the first + target. Set a capability from what the builder does, not what the database + could: the suite still generates an unsupported write and requires the builder + to refuse it. A database whose writes are asynchronous (ClickHouse mutations) + must make them visible to the next read (`mutations_sync=2`). - `targets`: the databases it runs on. Each target states its `Semantics`, the points where databases legitimately disagree (NULL ordering, integer division, what an unmatched LEFT JOIN column holds, LIKE case rules, …). The model reads diff --git a/tests/fuzz/build.ts b/tests/fuzz/build.ts index b52c86c..beb5df2 100644 --- a/tests/fuzz/build.ts +++ b/tests/fuzz/build.ts @@ -8,7 +8,7 @@ import { DateTime, Effect, Exit } from "effect" import type * as CH from "@maple-dev/effect-orm/clickhouse" import type { CoreContext } from "../core-cases" import type { FuzzDialect } from "./dialects/types" -import type { Agg, Cond, Expr, Operand, Plan } from "./plan" +import type { Agg, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" const pad = (n: number, width = 2) => String(n).padStart(width, "0") /** `YYYY-MM-DD hh:mm:ss.sss` in UTC: the string form a DateTime comparison accepts. */ @@ -18,13 +18,13 @@ export const timeString = (ms: number): string => { } /** Every param operand in the plan, in a fixed walk order, with the name it compiles as. */ -const nameParams = (plan: Plan): { readonly names: Map; readonly params: Record } => { +const nameParams = (plan: Plan, prefix: string): { readonly names: Map; readonly params: Record } => { const names = new Map() const params: Record = { orgId: plan.orgId } const operand = (op: Operand) => { if (op.k === "expr") return expr(op.e) if (op.k !== "param") return - const name = `p${names.size}` + const name = `${prefix}p${names.size}` names.set(op, name) params[name] = op.kind === "dateTime" ? DateTime.makeUnsafe(op.v as number) : op.v } @@ -74,9 +74,10 @@ export interface Built { readonly params: Record } -export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Built => { +/** `prefix` keeps the params of union branches apart: each branch names its own from `p0`. */ +export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan, prefix = ""): Built => { const { lit, param, and, or, not, rawCond, fromQuery } = dialect.module - const { names, params } = nameParams(plan) + const { names, params } = nameParams(plan, prefix) const expr = ($: any, e: Expr): any => { switch (e.k) { @@ -196,10 +197,9 @@ export type Compiled = * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that * escapes `compile` — or the builder calls before it — is `Threw`. */ -export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => { +export const classify = (compile: () => Effect.Effect, unknown>): Compiled => { try { - const { query, params } = buildPlan(dialect, ctx, plan) - const exit = Effect.runSyncExit(dialect.module.compile(query, params)) + const exit = Effect.runSyncExit(compile()) if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value } const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail") return failure !== undefined @@ -209,3 +209,26 @@ export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): return { _tag: "Threw", error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error) } } } + +export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => + classify(() => { + const { query, params } = buildPlan(dialect, ctx, plan) + return dialect.module.compile(query, params) + }) + +/** A union plan as `unionAll(...)` of its branches, each with its own params. */ +export const buildUnion = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Built => { + const built = plan.branches.map((branch, i) => buildPlan(dialect, ctx, branch, `b${i}`)) + let union: any = (dialect.module.unionAll as (...queries: ReadonlyArray) => unknown)(...built.map((b) => b.query)) + if (plan.orderBy.length > 0) union = union.orderBy(...plan.orderBy.map(([alias, dir]) => [alias, dir])) + if (plan.limit !== undefined) union = union.limit(plan.limit) + if (plan.offset !== undefined) union = union.offset(plan.offset) + if (plan.crossTenant) union = union.crossTenant() + return { query: union, params: Object.assign({}, ...built.map((b) => b.params)) } +} + +export const compileUnionPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Compiled => + classify(() => { + const { query, params } = buildUnion(dialect, ctx, plan) + return dialect.module.compileUnion(query, params) + }) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index 0ea4fba..a93962b 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -37,8 +37,17 @@ export const clickhouseText = async ( } } +/** + * A ClickHouse server bug, not the builder's: the common-expression rewrite can + * leave a one-argument and() (`a AND ((b AND a) OR a)`, `(a OR 0 OR a) AND (a OR 0)`, + * and more shapes than a matcher keeps up with), which fails with this error. + * The query is run again with the rewrite off, and its rows still checked. + */ +const SERVER_AND_BUG = /Number of arguments for function "and" should be at least 2/ + const runClickHouse = async (compiled: CH.CompiledQuery, settings: Record): Promise => { - const { ok, text } = await clickhouseText(compiled.sql, settings) + let { ok, text } = await clickhouseText(compiled.sql, settings) + if (!ok && SERVER_AND_BUG.test(text)) ({ ok, text } = await clickhouseText(compiled.sql, { ...settings, optimize_extract_common_expressions: "0" })) if (!ok) return { _tag: "DatabaseError", message: text.trim() } const wire = text .split("\n") @@ -58,12 +67,20 @@ const semantics: Semantics = { likeIgnoresCase: false, } +/** + * An UPDATE is an `ALTER TABLE … UPDATE` mutation, which runs in the background + * unless asked to wait, and a lightweight DELETE may too. Waiting makes a write + * visible to the read that follows it, as the writes suite needs. + */ +const syncWrites = { mutations_sync: "2", lightweight_deletes_sync: "2" } + const target = (name: string, joinUseNulls: boolean): FuzzTarget => ({ name, available: clickhouseEndpoint !== undefined, semantics: joinUseNulls ? { ...semantics, missingJoinColumn: "null" } : semantics, run: (compiled) => runClickHouse(compiled, { + ...syncWrites, join_use_nulls: joinUseNulls ? "1" : "0", output_format_json_quote_64bit_integers: joinUseNulls ? "1" : "0", }), @@ -132,5 +149,26 @@ export const clickhouse: FuzzDialect = { strings, ilike: true, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: false, + returning: false, + defaultCells: true, + scratch: { + table: (name) => + CH.table(name, { + external: true, + columns: { Id: CH.int64, OrgId: CH.string, Name: CH.string, Note: CH.nullable(CH.string), Qty: CH.column(CH.int64, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE ${name} (Id Int64, OrgId String, Name String, Note Nullable(String), Qty Int64 DEFAULT 5) ENGINE = MergeTree ORDER BY Id`], + drop: (name) => `DROP TABLE IF EXISTS ${name}`, + exec: async (sql) => { + const { ok, text } = await clickhouseText(sql, syncWrites) + if (!ok) throw new Error(text.trim()) + }, + }, + }, targets: [target("clickhouse", false), target("clickhouse-join-nulls", true)], } diff --git a/tests/fuzz/dialects/postgres.ts b/tests/fuzz/dialects/postgres.ts index cb10bdc..cd5cb22 100644 --- a/tests/fuzz/dialects/postgres.ts +++ b/tests/fuzz/dialects/postgres.ts @@ -69,6 +69,26 @@ export const postgres: FuzzDialect = { : `placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${count} parameters` }, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: true, + returning: true, + defaultCells: true, + scratch: { + table: (name) => + PG.table(name, { + external: true, + columns: { Id: PG.int8, OrgId: PG.text, Name: PG.text, Note: PG.nullable(PG.text), Qty: PG.column(PG.int8, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE "${name}" ("Id" int8 PRIMARY KEY, "OrgId" text NOT NULL, "Name" text NOT NULL, "Note" text, "Qty" int8 NOT NULL DEFAULT 5)`], + drop: (name) => `DROP TABLE IF EXISTS "${name}"`, + exec: async (sql) => { + await database().exec(sql) + }, + }, + }, targets: [ { name: "postgres", diff --git a/tests/fuzz/dialects/types.ts b/tests/fuzz/dialects/types.ts index 3e23e06..51ef8d8 100644 --- a/tests/fuzz/dialects/types.ts +++ b/tests/fuzz/dialects/types.ts @@ -81,6 +81,41 @@ export interface DialectModule { readonly not: typeof CH.not readonly rawCond: typeof CH.rawCond readonly fromQuery: typeof CH.fromQuery + readonly unionAll: typeof CH.unionAll + readonly compileUnion: (union: any, params?: Record) => Effect.Effect, unknown> + readonly insertInto: typeof CH.insertInto + readonly update: typeof CH.update + readonly deleteFrom: typeof CH.deleteFrom +} + +/** + * Which writes the dialect's builder compiles. A write it does not support is + * still generated, and must be refused at compile time rather than run. + */ +export interface WriteSupport { + /** `insertInto(t).select(query)`. */ + readonly insertSelect: boolean + readonly update: boolean + readonly delete: boolean + /** `onConflictDoNothing` / `onConflictDoUpdate`. */ + readonly onConflict: boolean + /** `.returning()` on INSERT, UPDATE and DELETE. */ + readonly returning: boolean + /** A multi-row VALUES where some rows leave out a defaulted column (`DEFAULT` per cell). */ + readonly defaultCells: boolean + /** + * The writes suite's scratch table, on the dialect's first target: a key + * `Id` (integer, unique), `OrgId` and `Name` (text), `Note` (nullable text) + * and `Qty` (integer, database default 5, so an insert may leave it out). + */ + readonly scratch: { + readonly table: (name: string) => CH.Table + /** Statements that create the table, empty. */ + readonly create: (name: string) => ReadonlyArray + readonly drop: (name: string) => string + /** Run one statement on the first target, failing if the database does. */ + readonly exec: (sql: string) => Promise + } } export interface FuzzDialect { @@ -99,7 +134,9 @@ export interface FuzzDialect { readonly checkPlaceholders?: (sql: string, parameterCount: number) => string | undefined /** Column types the literal round-trip property covers. */ readonly literals: ReadonlyArray - /** The databases this dialect's SQL runs on. The first is the literal suite's. */ + /** What the writes suite may generate, and how it gets a table to write to. */ + readonly writes: WriteSupport + /** The databases this dialect's SQL runs on. The first is the literal and writes suites'. */ readonly targets: ReadonlyArray /** Release a connection or process the targets opened. */ readonly close?: () => Promise diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index f170eb8..e47e536 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -41,7 +41,7 @@ export const knownHostileIssue = (input: HostileInput): string | undefined => // Query plans (queries.fuzz.test.ts) -import type { Plan } from "./plan" +import type { Plan, UnionPlan } from "./plan" export interface PlanInput { /** The target name (`dialects/*.ts`), or the dialect name for compile-only properties. */ @@ -49,43 +49,17 @@ export interface PlanInput { readonly plan: Plan } -/** Every object in a plan, depth-first: plans are plain data, so this reaches every expression. */ -const nodes = function* (value: unknown): Generator> { - if (Array.isArray(value)) for (const item of value) yield* nodes(item) - else if (value !== null && typeof value === "object") { - yield value as Record - for (const item of Object.values(value)) yield* nodes(item) - } +const planIssues: ReadonlyArray> = [] + +export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) + +// Unions (queries.fuzz.test.ts) + +export interface UnionInput { + readonly target: string + readonly plan: UnionPlan } -/** The conjuncts a WHERE list ANDs together, flattening nested ANDs. */ -const conjuncts = (conds: ReadonlyArray): Array => - conds.flatMap((c: any) => (c?.k === "and" ? conjuncts(c.cs) : [c])) -const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) - -const planIssues: ReadonlyArray> = [ - { - id: "clickhouse-server-and-absorption", - reason: - "A ClickHouse 25.8 server bug, not the builder: `WHERE a AND ((b AND a) OR a)` or `a AND ((a OR a) OR a)` fails with " + - "`Number of arguments for function \"and\" should be at least 2` — the common-expression rewrite leaves a " + - "one-argument and(). `SETTINGS optimize_extract_common_expressions = 0` avoids it.", - matches: ({ target, plan }) => { - if (!target.startsWith("clickhouse")) return false - const top = conjuncts(plan.where) - const disjuncts = (c: any): Array => (c?.k === "or" ? c.cs.flatMap(disjuncts) : [c]) - // Fails when every disjunct of a nested OR contains a top-level conjunct `w` (as `w`, or inside an AND): - // `w AND ((x AND w) OR w)`, `w AND ((w OR w) OR w)`. `w AND (w OR w)` and `w AND ((x AND w) OR y)` run. - return [...nodes(plan)].some( - (node) => - node.k === "or" && - (node.cs as Array).some((d) => d?.k === "or" || d?.k === "and") && - top.some((w) => disjuncts(node).every((d) => same(d, w) || (d?.k === "and" && conjuncts(d.cs).some((x) => same(x, w))))), - ) - }, - }, - - -] +const unionIssues: ReadonlyArray> = [] -export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) +export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) diff --git a/tests/fuzz/model.ts b/tests/fuzz/model.ts index 68f7889..b04ab13 100644 --- a/tests/fuzz/model.ts +++ b/tests/fuzz/model.ts @@ -12,7 +12,7 @@ import { fixtureRows } from "../core-cases" import type { Semantics } from "./dialects/types" import { skip } from "./harness" -import type { Agg, AggShape, ArithOp, Cond, Expr, Operand, Plan, RowsShape } from "./plan" +import type { Agg, ArithOp, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" type V = number | string | null type Bool = boolean | null @@ -69,7 +69,7 @@ const isTime = (e: Expr) => e.k === "col" && e.col === "Created" * database does (`-7 % toUInt32(3)` is 0), so the model declines those. */ type ChKind = "unsigned" | "signed" | "float" -const chKind = (e: Expr | number): ChKind => { +export const chKind = (e: Expr | number): ChKind => { if (typeof e === "number") return !Number.isInteger(e) ? "float" : e >= 0 ? "unsigned" : "signed" switch (e.k) { case "col": @@ -310,7 +310,7 @@ const sortRows = (env: Env, rows: Array, orderBy: ReadonlyArray(rows: ReadonlyArray, shape: RowsShape | AggShape): ReadonlyArray => { +const page = (rows: ReadonlyArray, shape: { readonly limit?: number | undefined; readonly offset?: number | undefined }): ReadonlyArray => { const start = shape.offset ?? 0 return rows.slice(start, shape.limit === undefined ? undefined : start + shape.limit) } @@ -393,3 +393,18 @@ export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { sortRows(env, rows, shape.orderBy) return { rows: page(rows, shape), ordered: total && !plan.wrap, orgs } } + +/** + * A union: each branch's rows (its own ORDER BY and LIMIT applied first), + * concatenated, then the union's ORDER BY, LIMIT and OFFSET. Rows compare as + * a list only when the union orders by every alias. + */ +export const evaluateUnion = (plan: UnionPlan, sem: Semantics): ModelResult => { + const parts = plan.branches.map((branch) => evaluate(branch, sem)) + const rows = parts.flatMap((part) => [...part.rows]) + const first = plan.branches[0]! + const aliases = first.shape.k === "rows" ? first.shape.select.map(([alias]) => alias) : [] + sortRows({ sem, plan: first, joined: false }, rows, plan.orderBy) + const total = plan.orderBy.length > 0 && aliases.every((alias) => plan.orderBy.some(([key]) => key === alias)) + return { rows: page(rows, plan), ordered: total, orgs: new Set(parts.flatMap((part) => [...part.orgs])) } +} diff --git a/tests/fuzz/plan.ts b/tests/fuzz/plan.ts index 3e9c7a3..e1ba22a 100644 --- a/tests/fuzz/plan.ts +++ b/tests/fuzz/plan.ts @@ -339,17 +339,85 @@ const aggShape = (joined: boolean, options: GenOptions): fc.Arbitrary })) }) +const orgIdArb = fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }) +const joinArb = fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }) + +// Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. +const whereArb = (joined: boolean, options: GenOptions): fc.Arbitrary> => + fc.oneof( + fc.constant([]), + fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), + ) + +/** + * `unionAll(branch, …)` with the union's own ORDER BY, LIMIT and OFFSET. Each + * branch is a row plan selecting the same aliases, each alias of one sort in + * every branch, so the union is well-typed by construction; a branch may have + * its own join, filter, DISTINCT, ORDER BY and LIMIT. + */ +export interface UnionPlan { + readonly orgId: string + readonly branches: ReadonlyArray + readonly orderBy: ReadonlyArray + readonly limit?: number + readonly offset?: number + readonly crossTenant: boolean +} + +const branch = (orgId: string, sorts: ReadonlyArray, options: GenOptions): fc.Arbitrary => + joinArb.chain((join) => { + const joined = join !== undefined + return fc.tuple(...sorts.map((sort) => expr(sort, joined, 2, options))).chain((exprs) => { + const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] + return fc.record({ where: whereArb(joined, options), distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + ({ where, distinct, order }): Plan => ({ orgId, join, where, shape: { k: "rows", select, distinct, ...order }, wrap: false, crossTenant: false }), + ) + }) + }) + +/** + * The union's ordering. Rows of different branches can tie on every alias but + * one, so it pages only when ordered by every alias: then tied rows are equal + * rows, and which of them LIMIT keeps does not matter. + */ +const unionOrdering = (aliases: ReadonlyArray) => + fc.oneof( + fc.constant({ orderBy: [] as ReadonlyArray, limit: undefined, offset: undefined }), + fc + .record({ + keys: fc.shuffledSubarray([...aliases], { minLength: aliases.length }), + dirs: fc.array(fc.constantFrom("asc" as const, "desc" as const), { minLength: aliases.length, maxLength: aliases.length }), + limit: fc.option(fc.integer({ min: 0, max: 8 }), { nil: undefined }), + offset: fc.option(fc.integer({ min: 0, max: 4 }), { nil: undefined }), + }) + .map(({ keys, dirs, limit, offset }) => ({ orderBy: keys.map((k, i) => [k, dirs[i]!] as const), limit, offset })), + fc.shuffledSubarray([...aliases], { minLength: 1 }).map((keys) => ({ orderBy: keys.map((k) => [k, "asc" as const] as const), limit: undefined, offset: undefined })), + ) + +export const unionPlan = (options: GenOptions): fc.Arbitrary => + fc + .record({ + orgId: orgIdArb, + sorts: fc.array(fc.constantFrom("num", "str", "nstr", "time"), { maxLength: 3 }), + branches: fc.integer({ min: 2, max: 3 }), + crossTenant: fc.boolean(), + }) + .chain(({ orgId, sorts, branches, crossTenant }) => + fc + .record({ + branches: fc.tuple(...Array.from({ length: branches }, () => branch(orgId, sorts, options))), + order: unionOrdering(["id", ...sorts.map((_, i) => `e${i}`)]), + }) + .map(({ branches, order }): UnionPlan => ({ orgId, branches, crossTenant, ...order })), + ) + export const plan = (options: GenOptions): fc.Arbitrary => - fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }).chain((join) => { + joinArb.chain((join) => { const joined = join !== undefined return fc.record({ - orgId: fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }), + orgId: orgIdArb, join: fc.constant(join), - // Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. - where: fc.oneof( - fc.constant([]), - fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), - ), + where: whereArb(joined, options), shape: fc.oneof({ weight: 3, arbitrary: rowsShape(joined, options) }, { weight: 2, arbitrary: aggShape(joined, options) }), wrap: fc.constantFrom(false as const, "all" as const, "names" as const), crossTenant: fc.boolean(), diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index 49730f1..890cc31 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -5,15 +5,18 @@ // // tenant soundness a query that reports "single-tenant" read rows of one org only // +// Unions are fuzzed the same way: `unionAll` of row plans, each branch with its +// own ORDER BY and LIMIT, under the union's own ORDER BY, LIMIT and OFFSET. +// // Every target in the registry (dialects/index.ts) runs; one whose database is // not reachable here is skipped, not dropped, so the run says so. import { afterAll, describe, expect, it } from "vitest" -import { compilePlan, scopeContext } from "./build" +import { compilePlan, compileUnionPlan, scopeContext, type Compiled } from "./build" import { closeAll, targets, type FuzzDialect, type FuzzTarget } from "./dialects" import { FuzzFailure, property, skipKnown, withContext } from "./harness" -import { knownPlanIssue } from "./known-issues" -import { evaluate } from "./model" -import { nastyOptions, plainOptions, plan, type Plan } from "./plan" +import { knownPlanIssue, knownUnionIssue } from "./known-issues" +import { evaluate, evaluateUnion, type ModelResult } from "./model" +import { nastyOptions, plainOptions, plan, unionPlan, type Plan, type UnionPlan } from "./plan" import { normalize } from "./values" afterAll(closeAll) @@ -27,15 +30,34 @@ const canonical = (rows: ReadonlyArray): Array => ) const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort() -const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { +const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - const expected = evaluate(input, target.semantics) - const result = compilePlan(dialect, dialect.context, input) + return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input), input.where) +} + +const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { + const knownUnion = knownUnionIssue({ target: target.name, plan: input }) + if (knownUnion !== undefined) skipKnown(knownUnion) + for (const branch of input.branches) { + const known = knownPlanIssue({ target: target.name, plan: branch }) + if (known !== undefined) skipKnown(known) + } + return checkRows( + target, + input.orgId, + evaluateUnion(input, target.semantics), + compileUnionPlan(dialect, dialect.context, input), + compileUnionPlan(dialect, scopeContext(dialect), input), + input.branches.map((branch) => branch.where), + ) +} + +/** Run what compiled and compare its rows with the model's; `scoped` is the same input over the plain tables. */ +const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled, where: unknown) => { if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. - const scoped = compilePlan(dialect, scopeContext(dialect), input) const tenantScope = scoped._tag === "Compiled" ? scoped.compiled.tenantScope : undefined const outcome = await target.run(compiled) await withContext({ sql: compiled.sql, parameters: compiled.parameters, tenantScope, outcome, expected: expected.rows }, () => { @@ -46,7 +68,10 @@ const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Pla if (expected.ordered) expect(actual).toEqual(want) else expect(sorted(actual)).toEqual(sorted(want)) if (tenantScope === "single-tenant") { - const foreign = [...expected.orgs].filter((org) => org !== input.orgId) + // The tenant is the `orgId` param, unless the plan also constrains OrgId itself (`OrgId IN ('org_2')` + // pins the query to org_2): then it is single-tenant by that, and any one org is right. + const pinnedElsewhere = /"col":"OrgId"/.test(JSON.stringify(where)) + const foreign = pinnedElsewhere ? [...expected.orgs].slice(1) : [...expected.orgs].filter((org) => org !== orgId) expect(foreign, "a single-tenant query read another org's rows").toEqual([]) } }) @@ -63,3 +88,15 @@ describe("queries match the model", () => { } } }) + +describe("unions match the model", () => { + for (const { dialect, target } of targets) { + for (const [label, options] of [ + ["plain strings", plainOptions(dialect)], + ["nasty strings", nastyOptions(dialect)], + ] as const) { + const name = `unions match the model (${target.name}, ${label})` + it.skipIf(!target.available)(name, () => property(name, unionPlan(options), (input) => unionMatchesModel(dialect, target, input), { runs: 60 }), 300_000) + } + } +}) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index a21260e..5f0b92d 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -5,7 +5,7 @@ // target at once). Inputs that hit an open entry in `known-issues.ts` are // skipped until the bug is fixed, then replayed for good. import { DateTime } from "effect" -import type { Plan, RowsShape } from "./plan" +import type { Plan, RowsShape, UnionPlan } from "./plan" const rows = (shape: RowsShape): Plan => ({ orgId: "org_1", join: undefined, where: [], shape, wrap: false, crossTenant: false }) @@ -28,7 +28,22 @@ export const pinned: Record> = { }), // fromquery-select-all. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, - // clickhouse-server-and-absorption. + // The same server bug in a shape the old matcher missed (ClickHouse 26.2): `(a OR 0 OR a) AND (a OR 0)`. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + orgId: "org_3", + where: [ + { + k: "and", + style: "method", + cs: [ + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "const", v: false }, { k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }] }, + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "in", neg: false, e: { k: "str", v: "" }, values: [] }] }, + ], + }, + ], + }, + // A ClickHouse server bug (a one-argument and()): run again without the rewrite, rows still checked (dialects/clickhouse.ts). { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), where: [ @@ -36,6 +51,11 @@ export const pinned: Record> = { { k: "or", style: "method", cs: [{ k: "and", style: "method", cs: [{ k: "const", v: true }, { k: "tenant", side: "orders" }] }, { k: "tenant", side: "orders" }] }, ], }, + // `OrgId IN ('org_2')` pins the query to org_2, not to the orgId param: single-tenant, and right to read org_2. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + where: [{ k: "in", neg: false, e: { k: "col", col: "OrgId" }, values: ["org_2"] }], + }, // pg-fraction-bound-to-integer. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), @@ -51,6 +71,20 @@ export const pinned: Record> = { "postgres:text[] select": [[]], "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + // Unions. + "unions match the model": [ + // clickhouse-union-variant-order-by: `Id + 1` (UInt64) under `Id + -1` (Int64), ordered by that column. + { + orgId: "org_1", + branches: [ + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: 1 }]], distinct: false, orderBy: [] }), + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: -1 }]], distinct: false, orderBy: [] }), + ], + orderBy: [["e0", "asc"], ["id", "asc"]], + crossTenant: false, + }, + ] satisfies ReadonlyArray, + // Hostile values. "hostile values are refused, never thrown (clickhouse)": [ { dialect: "clickhouse", column: "OrgId", site: "eq", value: 0n }, diff --git a/tests/fuzz/writes.fuzz.test.ts b/tests/fuzz/writes.fuzz.test.ts new file mode 100644 index 0000000..409cad9 --- /dev/null +++ b/tests/fuzz/writes.fuzz.test.ts @@ -0,0 +1,355 @@ +// Writes: random sequences of builder writes leave the table the model says. +// +// Each input gets a fresh scratch table on the dialect's first target (see +// `WriteSupport.scratch`), then runs a sequence of writes through the builder: +// multi-row inserts that leave defaulted or nullable columns out, INSERT … +// SELECT, ON CONFLICT DO NOTHING / DO UPDATE (with the existing and the +// proposed row), UPDATE and DELETE with a filter or `allRows()`, and RETURNING. +// After every write the table is read back and compared with a JS model of its +// rows, and RETURNING rows with the rows the model says the write touched. +// +// A write the dialect does not support (`WriteSupport`) is still generated, and +// must be refused at compile time: never run, never thrown. +import fc from "fast-check" +import { afterAll, describe, expect, it } from "vitest" +import { classify } from "./build" +import { closeAll, dialects, type FuzzDialect } from "./dialects" +import type { WriteSupport } from "./dialects/types" +import { FuzzFailure, property, skip, withContext } from "./harness" +import { nastyString, wellFormed } from "./values" + +afterAll(closeAll) + +// Plans + +interface Row { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note: string | null + readonly Qty: number +} +/** A row to insert; `Note` and `Qty` may be left out (NULL, and the database default 5). */ +interface NewRow { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note?: string | null + readonly Qty?: number +} +type QtySet = { readonly k: "value"; readonly v: number } | { readonly k: "add"; readonly v: number } | { readonly k: "excluded" } +interface SetPlan { + readonly Name?: string + readonly Note?: string | null + readonly Qty?: QtySet +} +type Where = + | { readonly k: "id"; readonly op: "eq" | "lt" | "gt"; readonly v: number } + | { readonly k: "name"; readonly v: string } + | { readonly k: "note"; readonly isNull: boolean } + | { readonly k: "org"; readonly v: string } + | { readonly k: "and" | "or"; readonly a: Where; readonly b: Where } +type Conflict = { readonly action: "nothing"; readonly target: boolean } | { readonly action: "update"; readonly set: SetPlan } +type WriteOp = + | { readonly k: "insert"; readonly rows: ReadonlyArray; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "insertSelect"; readonly offset: number; readonly where: Where; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "update"; readonly set: SetPlan; readonly where: Where | "all"; readonly returning: boolean } + | { readonly k: "delete"; readonly where: Where | "all"; readonly returning: boolean } +export interface WritesInput { + readonly ops: ReadonlyArray +} + +const DEFAULT_QTY = 5 +const ids = fc.integer({ min: 1, max: 12 }) +const org = fc.constantFrom("org_1", "org_2") + +const generators = (dialect: FuzzDialect) => { + const text = fc.oneof( + { weight: 3, arbitrary: fc.constantFrom("a", "b", "c", "it's", "x\\y") }, + { weight: 1, arbitrary: nastyString(dialect.strings).filter(wellFormed) }, + ) + const newRow: fc.Arbitrary = fc.record( + { Id: ids, OrgId: org, Name: text, Note: fc.option(text, { nil: null }), Qty: fc.integer({ min: -5, max: 50 }) }, + { requiredKeys: ["Id", "OrgId", "Name"] }, + ) + const qty = (excluded: boolean): fc.Arbitrary => + fc.oneof( + fc.integer({ min: -5, max: 50 }).map((v): QtySet => ({ k: "value", v })), + fc.integer({ min: -3, max: 3 }).map((v): QtySet => ({ k: "add", v })), + ...(excluded ? [fc.constant({ k: "excluded" })] : []), + ) + const set = (excluded: boolean): fc.Arbitrary => + fc + .record({ Name: text, Note: fc.option(text, { nil: null }), Qty: qty(excluded) }, { requiredKeys: [] }) + .filter((s) => Object.keys(s).length > 0) + const leaf: fc.Arbitrary = fc.oneof( + fc.record({ k: fc.constant("id" as const), op: fc.constantFrom("eq" as const, "lt" as const, "gt" as const), v: ids }), + fc.record({ k: fc.constant("name" as const), v: text }), + fc.record({ k: fc.constant("note" as const), isNull: fc.boolean() }), + fc.record({ k: fc.constant("org" as const), v: org }), + ) + const where: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: leaf }, + { weight: 1, arbitrary: fc.record({ k: fc.constantFrom("and" as const, "or" as const), a: leaf, b: leaf }) }, + ) + const conflict: fc.Arbitrary = fc.option( + fc.oneof( + fc.record({ action: fc.constant("nothing" as const), target: fc.boolean() }), + fc.record({ action: fc.constant("update" as const), set: set(true) }), + ), + { nil: undefined, freq: 2 }, + ) + const returning = fc.boolean() + const op: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: fc.record({ k: fc.constant("insert" as const), rows: fc.array(newRow, { minLength: 1, maxLength: 4 }), conflict, returning }, { requiredKeys: ["k", "rows", "returning"] }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("insertSelect" as const), offset: fc.constantFrom(100, 1000, 10_000), where, conflict, returning }, { requiredKeys: ["k", "offset", "where", "returning"] }) }, + { weight: 2, arbitrary: fc.record({ k: fc.constant("update" as const), set: set(false), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("delete" as const), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + ) + // The first write seeds rows, so later ones have something to change. + const seed = fc.array(newRow, { minLength: 1, maxLength: 5 }).map((rows): WriteOp => ({ k: "insert", rows, returning: false })) + return fc.tuple(seed, fc.array(op, { minLength: 1, maxLength: 5 })).map(([first, rest]): WritesInput => ({ ops: [first, ...rest] })) +} + +// The model + +const matches = (where: Where | "all", row: Row): boolean => { + if (where === "all") return true + switch (where.k) { + case "id": + return where.op === "eq" ? row.Id === where.v : where.op === "lt" ? row.Id < where.v : row.Id > where.v + case "name": + return row.Name === where.v + case "note": + return (row.Note === null) === where.isNull + case "org": + return row.OrgId === where.v + case "and": + return matches(where.a, row) && matches(where.b, row) + case "or": + return matches(where.a, row) || matches(where.b, row) + } +} + +const applySet = (set: SetPlan, row: Row, excluded?: Row): Row => ({ + ...row, + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined + ? {} + : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? row.Qty + set.Qty.v : excluded!.Qty }), +}) + +const fullRow = (row: NewRow): Row => ({ Id: row.Id, OrgId: row.OrgId, Name: row.Name, Note: row.Note ?? null, Qty: row.Qty ?? DEFAULT_QTY }) + +/** + * Rows inserted one by one, as the database does: a key that exists is a + * conflict, skipped or updated. Returns the rows the statement inserted or + * updated, which is what RETURNING gives. + */ +const insertRows = (state: Array, rows: ReadonlyArray, conflict: Conflict | undefined): Array => { + const touched: Array = [] + for (const row of rows) { + const at = state.findIndex((r) => r.Id === row.Id) + if (at === -1) { + state.push(row) + touched.push(row) + } else if (conflict?.action === "update") { + state[at] = applySet(conflict.set, state[at]!, row) + touched.push(state[at]!) + } + } + return touched +} + +/** Apply a write to the model; the rows RETURNING gives. */ +const apply = (state: Array, op: WriteOp): Array => { + switch (op.k) { + case "insert": + return insertRows(state, op.rows.map(fullRow), op.conflict) + case "insertSelect": + // The SELECT reads the table as it was before the statement. + return insertRows(state, state.filter((row) => matches(op.where, row)).map((row) => ({ ...row, Id: row.Id + op.offset })), op.conflict) + case "update": { + const touched: Array = [] + for (const [i, row] of state.entries()) { + if (!matches(op.where, row)) continue + state[i] = applySet(op.set, row) + touched.push(state[i]!) + } + return touched + } + case "delete": { + const removed = state.filter((row) => matches(op.where, row)) + const kept = state.filter((row) => !matches(op.where, row)) + state.splice(0, state.length, ...kept) + return removed + } + } +} + +/** + * Make a write valid for the table it meets: an insert without ON CONFLICT + * gets fresh keys where its keys exist (a duplicate key is an error, not a + * finding), and one with DO UPDATE no repeated key (Postgres refuses to update + * a row twice in one statement). + */ +const repair = (state: ReadonlyArray, used: Set, op: WriteOp): WriteOp => { + const fresh = () => { + let id = Math.max(0, ...used) + 1 + while (used.has(id)) id++ + return id + } + if (op.k === "insert") { + const seen = new Set() + const rows = op.rows.map((row) => { + const clash = op.conflict === undefined ? state.some((r) => r.Id === row.Id) || seen.has(row.Id) : op.conflict.action === "update" && seen.has(row.Id) + const id = clash ? fresh() : row.Id + seen.add(id) + used.add(id) + return id === row.Id ? row : { ...row, Id: id } + }) + return { ...op, rows } + } + if (op.k === "insertSelect" && op.conflict === undefined) { + const copies = state.filter((row) => matches(op.where, row)).map((row) => row.Id + op.offset) + if (copies.some((id) => state.some((r) => r.Id === id))) skip("INSERT … SELECT would repeat a key") + for (const id of copies) used.add(id) + } + if (op.k === "insertSelect") for (const row of state) used.add(row.Id + op.offset) + return op +} + +// Capabilities + +/** The first feature `op` uses that the dialect does not support, if any. */ +const unsupported = (support: WriteSupport, op: WriteOp): string | undefined => { + if (op.k === "insertSelect" && !support.insertSelect) return "INSERT … SELECT" + if (op.k === "update" && !support.update) return "UPDATE" + if (op.k === "delete" && !support.delete) return "DELETE" + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined && !support.onConflict) return "ON CONFLICT" + if (op.returning && !support.returning) return "RETURNING" + if (op.k === "insert" && !support.defaultCells && op.rows.length > 1) { + for (const column of ["Note", "Qty"] as const) { + const given = op.rows.filter((row) => Object.hasOwn(row, column)).length + if (given > 0 && given < op.rows.length) return `DEFAULT for a missing ${column} in a multi-row VALUES` + } + } + return undefined +} + +// Builder calls + +const whereCond = ($: any, where: Where): any => { + switch (where.k) { + case "id": + return $.Id[where.op](where.v) + case "name": + return $.Name.eq(where.v) + case "note": + return where.isNull ? $.Note.isNull() : $.Note.isNotNull() + case "org": + return $.OrgId.eq(where.v) + case "and": + case "or": + return whereCond($, where.a)[where.k](whereCond($, where.b)) + } +} + +const setRecord = (set: SetPlan, $: any, excluded?: any): Record => ({ + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined ? {} : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? $.Qty.add(set.Qty.v) : excluded.Qty }), +}) + +const buildWrite = (dialect: FuzzDialect, table: any, op: WriteOp): any => { + const { insertInto, update, deleteFrom, from } = dialect.module + let write: any + const filtered = (w: any, where: Where | "all") => (where === "all" ? w.allRows() : w.where(($: any) => [whereCond($, where)])) + switch (op.k) { + case "insert": + write = insertInto(table).values(op.rows.length === 1 ? (op.rows[0] as never) : (op.rows as never)) + break + case "insertSelect": + write = insertInto(table).select( + (from(table) as any) + .select(($: any) => ({ Id: $.Id.add(op.offset), OrgId: $.OrgId, Name: $.Name, Note: $.Note, Qty: $.Qty })) + .where(($: any) => [whereCond($, op.where)]), + ) + break + case "update": + write = filtered(update(table).set(($: any) => setRecord(op.set, $)), op.where) + break + case "delete": + write = filtered(deleteFrom(table), op.where) + break + } + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined) { + const conflict = op.conflict + write = + conflict.action === "nothing" + ? write.onConflictDoNothing(conflict.target ? { target: ["Id"] } : {}) + : write.onConflictDoUpdate({ target: ["Id"], set: ($: any, excluded: any) => setRecord(conflict.set, $, excluded) }) + } + return op.returning ? write.returning() : write +} + +// The property + +const byId = (rows: ReadonlyArray) => [...rows].sort((a, b) => a.Id - b.Id) +const asRows = (rows: ReadonlyArray): Array => + rows.map((row) => { + const r = row as Record + return { Id: Number(r.Id), OrgId: r.OrgId as string, Name: r.Name as string, Note: (r.Note ?? null) as string | null, Qty: Number(r.Qty) } + }) + +let tables = 0 + +const writesMatchModel = async (dialect: FuzzDialect, input: WritesInput) => { + const { scratch } = dialect.writes + const target = dialect.targets[0]! + const name = `fuzz_writes_${process.pid}_${tables++}` + for (const sql of scratch.create(name)) await scratch.exec(sql) + try { + const table = scratch.table(name) + const read = classify(() => dialect.module.compile((dialect.module.from(table) as any).select(), {})) + if (read._tag !== "Compiled") throw new FuzzFailure("the scratch table's SELECT did not compile", { error: read.error }) + const state: Array = [] + const used = new Set() + for (const [step, planned] of input.ops.entries()) { + const op = repair(state, used, planned) + const missing = unsupported(dialect.writes, op) + const result = classify(() => dialect.module.compile(buildWrite(dialect, table, op), {})) + const context = { step, op, compiled: result._tag === "Compiled" ? { sql: result.compiled.sql, parameters: result.compiled.parameters } : result, before: byId(state) } + if (result._tag === "Threw") throw new FuzzFailure("the builder threw instead of failing", context) + if (missing !== undefined) { + if (result._tag === "Compiled") throw new FuzzFailure(`${missing} compiled for ${dialect.name}, which does not support it`, context) + // A clause the dialect lacks fails with a typed error, or dies with a QueryBuilderDefect (src/ch/errors.ts); nothing else. + if (result._tag !== "Refused" && !(result._tag === "Defect" && result.error.includes("QueryBuilderDefect"))) { + throw new FuzzFailure(`${missing} was not refused with a QueryBuilderError or QueryBuilderDefect`, context) + } + continue + } + if (result._tag !== "Compiled") throw new FuzzFailure(`a supported write was not compiled (${result._tag})`, context) + const touched = apply(state, op) + const outcome = await target.run(result.compiled) + const after = await target.run(read.compiled) + await withContext({ ...context, outcome, expected: byId(state), returningExpected: op.returning ? byId(touched) : undefined, after }, () => { + expect(outcome._tag).toBe("Rows") + expect(after._tag).toBe("Rows") + if (outcome._tag !== "Rows" || after._tag !== "Rows") return + expect(byId(asRows(after.rows))).toEqual(byId(state)) + if (op.returning) expect(byId(asRows(outcome.rows))).toEqual(byId(touched)) + }) + } + } finally { + await scratch.exec(scratch.drop(name)) + } +} + +describe("writes match the model", () => { + for (const dialect of dialects) { + const name = `writes match the model (${dialect.name})` + it.skipIf(!dialect.targets[0]!.available)(name, () => property(name, generators(dialect), (input) => writesMatchModel(dialect, input), { runs: 40 }), 300_000) + } +})