From 8184ac422f0beafd432b6739784132dc2c29a474 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:45:52 +0100 Subject: [PATCH 1/8] test(fuzz): fuzz unions and writes on every dialect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unions: `unionAll` of two or three row plans that select the same aliases (each alias one sort in every branch), each branch with its own join, filter, DISTINCT, ORDER BY and LIMIT, under the union's ORDER BY, LIMIT and OFFSET. The model concatenates the branches' rows and pages them; the union pages only when it orders by every alias, so ties are equal rows. New properties `unions match the model (, …)`. Writes: a new suite. Each input gets a fresh scratch table on the dialect's first target and runs a sequence of builder writes: multi-row inserts leaving defaulted or nullable columns out, INSERT … SELECT, ON CONFLICT DO NOTHING / DO UPDATE (with the existing and the excluded row), UPDATE and DELETE with a filter or allRows(), and RETURNING. After every write the table is read back and compared with a JS model of its rows, and RETURNING with the rows the write touched. A write the dialect does not support is still generated and must be refused at compile time. `FuzzDialect.writes` declares the capabilities and the scratch table (query `Table`, create/drop DDL, `exec`); `DialectModule` gains unionAll, compileUnion, insertInto, update and deleteFrom. ClickHouse targets pass mutations_sync=2 and lightweight_deletes_sync=2 so an ALTER TABLE UPDATE or DELETE is visible to the next read. Found: a ClickHouse 26 UNION ALL column whose branches have numeric types with no common supertype (UInt64 and Int64, Int64 and Float64) becomes a Variant, and ORDER BY on it fails. Recorded in known-issues (clickhouse-union-variant-order-by) and pinned. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/README.md | 11 +- tests/fuzz/build.ts | 39 +++- tests/fuzz/dialects/clickhouse.ts | 29 +++ tests/fuzz/dialects/postgres.ts | 20 ++ tests/fuzz/dialects/types.ts | 39 +++- tests/fuzz/known-issues.ts | 39 +++- tests/fuzz/model.ts | 21 +- tests/fuzz/plan.ts | 82 ++++++- tests/fuzz/queries.fuzz.test.ts | 51 ++++- tests/fuzz/regressions.ts | 16 +- tests/fuzz/writes.fuzz.test.ts | 351 ++++++++++++++++++++++++++++++ 11 files changed, 667 insertions(+), 31 deletions(-) create mode 100644 tests/fuzz/writes.fuzz.test.ts diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 25a6b4f..4c34553 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -17,7 +17,8 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | File | Property | Oracle | | --- | --- | --- | -| `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target | +| `queries.fuzz.test.ts` | random plans, and `unionAll`s of them (branch and union ORDER BY / LIMIT / OFFSET), return the rows `model.ts` computes | a JS model of SQL, per target | +| `writes.fuzz.test.ts` | random sequences of inserts (multi-row, defaulted cells, INSERT … SELECT, ON CONFLICT), updates and deletes, with RETURNING, leave a scratch table as the model says; unsupported writes are refused | a JS model of the table's rows | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | @@ -36,6 +37,14 @@ A `FuzzDialect` gives: - `strings` and `ilike`: what the generators may produce for it. - `literals`: its column types, each with a way to build the reference row *without* the builder (usually the database parsing JSON). +- `writes`: which writes its builder compiles (`insertSelect`, `update`, `delete`, + `onConflict`, `returning`, `defaultCells` for a multi-row VALUES that leaves a + defaulted column out of some rows), and a `scratch` table: its query `Table`, + the DDL that creates and drops it, and `exec` to run that DDL on the first + target. Set a capability from what the builder does, not what the database + could: the suite still generates an unsupported write and requires the builder + to refuse it. A database whose writes are asynchronous (ClickHouse mutations) + must make them visible to the next read (`mutations_sync=2`). - `targets`: the databases it runs on. Each target states its `Semantics`, the points where databases legitimately disagree (NULL ordering, integer division, what an unmatched LEFT JOIN column holds, LIKE case rules, …). The model reads diff --git a/tests/fuzz/build.ts b/tests/fuzz/build.ts index b52c86c..beb5df2 100644 --- a/tests/fuzz/build.ts +++ b/tests/fuzz/build.ts @@ -8,7 +8,7 @@ import { DateTime, Effect, Exit } from "effect" import type * as CH from "@maple-dev/effect-orm/clickhouse" import type { CoreContext } from "../core-cases" import type { FuzzDialect } from "./dialects/types" -import type { Agg, Cond, Expr, Operand, Plan } from "./plan" +import type { Agg, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" const pad = (n: number, width = 2) => String(n).padStart(width, "0") /** `YYYY-MM-DD hh:mm:ss.sss` in UTC: the string form a DateTime comparison accepts. */ @@ -18,13 +18,13 @@ export const timeString = (ms: number): string => { } /** Every param operand in the plan, in a fixed walk order, with the name it compiles as. */ -const nameParams = (plan: Plan): { readonly names: Map; readonly params: Record } => { +const nameParams = (plan: Plan, prefix: string): { readonly names: Map; readonly params: Record } => { const names = new Map() const params: Record = { orgId: plan.orgId } const operand = (op: Operand) => { if (op.k === "expr") return expr(op.e) if (op.k !== "param") return - const name = `p${names.size}` + const name = `${prefix}p${names.size}` names.set(op, name) params[name] = op.kind === "dateTime" ? DateTime.makeUnsafe(op.v as number) : op.v } @@ -74,9 +74,10 @@ export interface Built { readonly params: Record } -export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Built => { +/** `prefix` keeps the params of union branches apart: each branch names its own from `p0`. */ +export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan, prefix = ""): Built => { const { lit, param, and, or, not, rawCond, fromQuery } = dialect.module - const { names, params } = nameParams(plan) + const { names, params } = nameParams(plan, prefix) const expr = ($: any, e: Expr): any => { switch (e.k) { @@ -196,10 +197,9 @@ export type Compiled = * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that * escapes `compile` — or the builder calls before it — is `Threw`. */ -export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => { +export const classify = (compile: () => Effect.Effect, unknown>): Compiled => { try { - const { query, params } = buildPlan(dialect, ctx, plan) - const exit = Effect.runSyncExit(dialect.module.compile(query, params)) + const exit = Effect.runSyncExit(compile()) if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value } const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail") return failure !== undefined @@ -209,3 +209,26 @@ export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): return { _tag: "Threw", error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error) } } } + +export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => + classify(() => { + const { query, params } = buildPlan(dialect, ctx, plan) + return dialect.module.compile(query, params) + }) + +/** A union plan as `unionAll(...)` of its branches, each with its own params. */ +export const buildUnion = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Built => { + const built = plan.branches.map((branch, i) => buildPlan(dialect, ctx, branch, `b${i}`)) + let union: any = (dialect.module.unionAll as (...queries: ReadonlyArray) => unknown)(...built.map((b) => b.query)) + if (plan.orderBy.length > 0) union = union.orderBy(...plan.orderBy.map(([alias, dir]) => [alias, dir])) + if (plan.limit !== undefined) union = union.limit(plan.limit) + if (plan.offset !== undefined) union = union.offset(plan.offset) + if (plan.crossTenant) union = union.crossTenant() + return { query: union, params: Object.assign({}, ...built.map((b) => b.params)) } +} + +export const compileUnionPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Compiled => + classify(() => { + const { query, params } = buildUnion(dialect, ctx, plan) + return dialect.module.compileUnion(query, params) + }) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index 0ea4fba..f130294 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -58,12 +58,20 @@ const semantics: Semantics = { likeIgnoresCase: false, } +/** + * An UPDATE is an `ALTER TABLE … UPDATE` mutation, which runs in the background + * unless asked to wait, and a lightweight DELETE may too. Waiting makes a write + * visible to the read that follows it, as the writes suite needs. + */ +const syncWrites = { mutations_sync: "2", lightweight_deletes_sync: "2" } + const target = (name: string, joinUseNulls: boolean): FuzzTarget => ({ name, available: clickhouseEndpoint !== undefined, semantics: joinUseNulls ? { ...semantics, missingJoinColumn: "null" } : semantics, run: (compiled) => runClickHouse(compiled, { + ...syncWrites, join_use_nulls: joinUseNulls ? "1" : "0", output_format_json_quote_64bit_integers: joinUseNulls ? "1" : "0", }), @@ -132,5 +140,26 @@ export const clickhouse: FuzzDialect = { strings, ilike: true, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: false, + returning: false, + defaultCells: true, + scratch: { + table: (name) => + CH.table(name, { + external: true, + columns: { Id: CH.int64, OrgId: CH.string, Name: CH.string, Note: CH.nullable(CH.string), Qty: CH.column(CH.int64, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE ${name} (Id Int64, OrgId String, Name String, Note Nullable(String), Qty Int64 DEFAULT 5) ENGINE = MergeTree ORDER BY Id`], + drop: (name) => `DROP TABLE IF EXISTS ${name}`, + exec: async (sql) => { + const { ok, text } = await clickhouseText(sql, syncWrites) + if (!ok) throw new Error(text.trim()) + }, + }, + }, targets: [target("clickhouse", false), target("clickhouse-join-nulls", true)], } diff --git a/tests/fuzz/dialects/postgres.ts b/tests/fuzz/dialects/postgres.ts index cb10bdc..cd5cb22 100644 --- a/tests/fuzz/dialects/postgres.ts +++ b/tests/fuzz/dialects/postgres.ts @@ -69,6 +69,26 @@ export const postgres: FuzzDialect = { : `placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${count} parameters` }, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: true, + returning: true, + defaultCells: true, + scratch: { + table: (name) => + PG.table(name, { + external: true, + columns: { Id: PG.int8, OrgId: PG.text, Name: PG.text, Note: PG.nullable(PG.text), Qty: PG.column(PG.int8, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE "${name}" ("Id" int8 PRIMARY KEY, "OrgId" text NOT NULL, "Name" text NOT NULL, "Note" text, "Qty" int8 NOT NULL DEFAULT 5)`], + drop: (name) => `DROP TABLE IF EXISTS "${name}"`, + exec: async (sql) => { + await database().exec(sql) + }, + }, + }, targets: [ { name: "postgres", diff --git a/tests/fuzz/dialects/types.ts b/tests/fuzz/dialects/types.ts index 3e23e06..51ef8d8 100644 --- a/tests/fuzz/dialects/types.ts +++ b/tests/fuzz/dialects/types.ts @@ -81,6 +81,41 @@ export interface DialectModule { readonly not: typeof CH.not readonly rawCond: typeof CH.rawCond readonly fromQuery: typeof CH.fromQuery + readonly unionAll: typeof CH.unionAll + readonly compileUnion: (union: any, params?: Record) => Effect.Effect, unknown> + readonly insertInto: typeof CH.insertInto + readonly update: typeof CH.update + readonly deleteFrom: typeof CH.deleteFrom +} + +/** + * Which writes the dialect's builder compiles. A write it does not support is + * still generated, and must be refused at compile time rather than run. + */ +export interface WriteSupport { + /** `insertInto(t).select(query)`. */ + readonly insertSelect: boolean + readonly update: boolean + readonly delete: boolean + /** `onConflictDoNothing` / `onConflictDoUpdate`. */ + readonly onConflict: boolean + /** `.returning()` on INSERT, UPDATE and DELETE. */ + readonly returning: boolean + /** A multi-row VALUES where some rows leave out a defaulted column (`DEFAULT` per cell). */ + readonly defaultCells: boolean + /** + * The writes suite's scratch table, on the dialect's first target: a key + * `Id` (integer, unique), `OrgId` and `Name` (text), `Note` (nullable text) + * and `Qty` (integer, database default 5, so an insert may leave it out). + */ + readonly scratch: { + readonly table: (name: string) => CH.Table + /** Statements that create the table, empty. */ + readonly create: (name: string) => ReadonlyArray + readonly drop: (name: string) => string + /** Run one statement on the first target, failing if the database does. */ + readonly exec: (sql: string) => Promise + } } export interface FuzzDialect { @@ -99,7 +134,9 @@ export interface FuzzDialect { readonly checkPlaceholders?: (sql: string, parameterCount: number) => string | undefined /** Column types the literal round-trip property covers. */ readonly literals: ReadonlyArray - /** The databases this dialect's SQL runs on. The first is the literal suite's. */ + /** What the writes suite may generate, and how it gets a table to write to. */ + readonly writes: WriteSupport + /** The databases this dialect's SQL runs on. The first is the literal and writes suites'. */ readonly targets: ReadonlyArray /** Release a connection or process the targets opened. */ readonly close?: () => Promise diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index f170eb8..0bf51b9 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -41,7 +41,8 @@ export const knownHostileIssue = (input: HostileInput): string | undefined => // Query plans (queries.fuzz.test.ts) -import type { Plan } from "./plan" +import { chKind } from "./model" +import type { Plan, UnionPlan } from "./plan" export interface PlanInput { /** The target name (`dialects/*.ts`), or the dialect name for compile-only properties. */ @@ -89,3 +90,39 @@ const planIssues: ReadonlyArray> = [ ] export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) + +// Unions (queries.fuzz.test.ts) + +export interface UnionInput { + readonly target: string + readonly plan: UnionPlan +} + +/** The expression a branch selects under `alias`. */ +const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) + +const unionIssues: ReadonlyArray> = [ + { + id: "clickhouse-union-variant-order-by", + reason: + "On ClickHouse 26, a UNION ALL column whose branches have numeric types with no common supertype (`Id + 1` is " + + "UInt64, `Id - 1` Int64; Int64 and Float64) becomes a Variant, and the union's ORDER BY on it fails with " + + "`Data types Variant/Dynamic are not allowed in ORDER BY keys`. Fix: have compileUnion cast a numeric union " + + "column to one type (e.g. Float64, or the widest signed type) when its branches' expressions differ.", + matches: ({ target, plan }) => + target.startsWith("clickhouse") && + plan.orderBy.some(([alias]) => { + const kinds = new Set( + plan.branches.map((branch) => { + const e = selected(branch, alias) + return e === undefined || (e.k !== "num" && e.k !== "arith" && !(e.k === "col" && (e.col === "Id" || e.col === "Amount"))) + ? "other" + : `${chKind(e)}:${e.k === "col" ? e.col : e.k}` + }), + ) + return kinds.size > 1 && !kinds.has("other") + }), + }, +] + +export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) diff --git a/tests/fuzz/model.ts b/tests/fuzz/model.ts index 68f7889..b04ab13 100644 --- a/tests/fuzz/model.ts +++ b/tests/fuzz/model.ts @@ -12,7 +12,7 @@ import { fixtureRows } from "../core-cases" import type { Semantics } from "./dialects/types" import { skip } from "./harness" -import type { Agg, AggShape, ArithOp, Cond, Expr, Operand, Plan, RowsShape } from "./plan" +import type { Agg, ArithOp, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" type V = number | string | null type Bool = boolean | null @@ -69,7 +69,7 @@ const isTime = (e: Expr) => e.k === "col" && e.col === "Created" * database does (`-7 % toUInt32(3)` is 0), so the model declines those. */ type ChKind = "unsigned" | "signed" | "float" -const chKind = (e: Expr | number): ChKind => { +export const chKind = (e: Expr | number): ChKind => { if (typeof e === "number") return !Number.isInteger(e) ? "float" : e >= 0 ? "unsigned" : "signed" switch (e.k) { case "col": @@ -310,7 +310,7 @@ const sortRows = (env: Env, rows: Array, orderBy: ReadonlyArray(rows: ReadonlyArray, shape: RowsShape | AggShape): ReadonlyArray => { +const page = (rows: ReadonlyArray, shape: { readonly limit?: number | undefined; readonly offset?: number | undefined }): ReadonlyArray => { const start = shape.offset ?? 0 return rows.slice(start, shape.limit === undefined ? undefined : start + shape.limit) } @@ -393,3 +393,18 @@ export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { sortRows(env, rows, shape.orderBy) return { rows: page(rows, shape), ordered: total && !plan.wrap, orgs } } + +/** + * A union: each branch's rows (its own ORDER BY and LIMIT applied first), + * concatenated, then the union's ORDER BY, LIMIT and OFFSET. Rows compare as + * a list only when the union orders by every alias. + */ +export const evaluateUnion = (plan: UnionPlan, sem: Semantics): ModelResult => { + const parts = plan.branches.map((branch) => evaluate(branch, sem)) + const rows = parts.flatMap((part) => [...part.rows]) + const first = plan.branches[0]! + const aliases = first.shape.k === "rows" ? first.shape.select.map(([alias]) => alias) : [] + sortRows({ sem, plan: first, joined: false }, rows, plan.orderBy) + const total = plan.orderBy.length > 0 && aliases.every((alias) => plan.orderBy.some(([key]) => key === alias)) + return { rows: page(rows, plan), ordered: total, orgs: new Set(parts.flatMap((part) => [...part.orgs])) } +} diff --git a/tests/fuzz/plan.ts b/tests/fuzz/plan.ts index 3e9c7a3..e1ba22a 100644 --- a/tests/fuzz/plan.ts +++ b/tests/fuzz/plan.ts @@ -339,17 +339,85 @@ const aggShape = (joined: boolean, options: GenOptions): fc.Arbitrary })) }) +const orgIdArb = fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }) +const joinArb = fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }) + +// Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. +const whereArb = (joined: boolean, options: GenOptions): fc.Arbitrary> => + fc.oneof( + fc.constant([]), + fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), + ) + +/** + * `unionAll(branch, …)` with the union's own ORDER BY, LIMIT and OFFSET. Each + * branch is a row plan selecting the same aliases, each alias of one sort in + * every branch, so the union is well-typed by construction; a branch may have + * its own join, filter, DISTINCT, ORDER BY and LIMIT. + */ +export interface UnionPlan { + readonly orgId: string + readonly branches: ReadonlyArray + readonly orderBy: ReadonlyArray + readonly limit?: number + readonly offset?: number + readonly crossTenant: boolean +} + +const branch = (orgId: string, sorts: ReadonlyArray, options: GenOptions): fc.Arbitrary => + joinArb.chain((join) => { + const joined = join !== undefined + return fc.tuple(...sorts.map((sort) => expr(sort, joined, 2, options))).chain((exprs) => { + const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] + return fc.record({ where: whereArb(joined, options), distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + ({ where, distinct, order }): Plan => ({ orgId, join, where, shape: { k: "rows", select, distinct, ...order }, wrap: false, crossTenant: false }), + ) + }) + }) + +/** + * The union's ordering. Rows of different branches can tie on every alias but + * one, so it pages only when ordered by every alias: then tied rows are equal + * rows, and which of them LIMIT keeps does not matter. + */ +const unionOrdering = (aliases: ReadonlyArray) => + fc.oneof( + fc.constant({ orderBy: [] as ReadonlyArray, limit: undefined, offset: undefined }), + fc + .record({ + keys: fc.shuffledSubarray([...aliases], { minLength: aliases.length }), + dirs: fc.array(fc.constantFrom("asc" as const, "desc" as const), { minLength: aliases.length, maxLength: aliases.length }), + limit: fc.option(fc.integer({ min: 0, max: 8 }), { nil: undefined }), + offset: fc.option(fc.integer({ min: 0, max: 4 }), { nil: undefined }), + }) + .map(({ keys, dirs, limit, offset }) => ({ orderBy: keys.map((k, i) => [k, dirs[i]!] as const), limit, offset })), + fc.shuffledSubarray([...aliases], { minLength: 1 }).map((keys) => ({ orderBy: keys.map((k) => [k, "asc" as const] as const), limit: undefined, offset: undefined })), + ) + +export const unionPlan = (options: GenOptions): fc.Arbitrary => + fc + .record({ + orgId: orgIdArb, + sorts: fc.array(fc.constantFrom("num", "str", "nstr", "time"), { maxLength: 3 }), + branches: fc.integer({ min: 2, max: 3 }), + crossTenant: fc.boolean(), + }) + .chain(({ orgId, sorts, branches, crossTenant }) => + fc + .record({ + branches: fc.tuple(...Array.from({ length: branches }, () => branch(orgId, sorts, options))), + order: unionOrdering(["id", ...sorts.map((_, i) => `e${i}`)]), + }) + .map(({ branches, order }): UnionPlan => ({ orgId, branches, crossTenant, ...order })), + ) + export const plan = (options: GenOptions): fc.Arbitrary => - fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }).chain((join) => { + joinArb.chain((join) => { const joined = join !== undefined return fc.record({ - orgId: fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }), + orgId: orgIdArb, join: fc.constant(join), - // Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. - where: fc.oneof( - fc.constant([]), - fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), - ), + where: whereArb(joined, options), shape: fc.oneof({ weight: 3, arbitrary: rowsShape(joined, options) }, { weight: 2, arbitrary: aggShape(joined, options) }), wrap: fc.constantFrom(false as const, "all" as const, "names" as const), crossTenant: fc.boolean(), diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index 49730f1..c836dd8 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -5,15 +5,18 @@ // // tenant soundness a query that reports "single-tenant" read rows of one org only // +// Unions are fuzzed the same way: `unionAll` of row plans, each branch with its +// own ORDER BY and LIMIT, under the union's own ORDER BY, LIMIT and OFFSET. +// // Every target in the registry (dialects/index.ts) runs; one whose database is // not reachable here is skipped, not dropped, so the run says so. import { afterAll, describe, expect, it } from "vitest" -import { compilePlan, scopeContext } from "./build" +import { compilePlan, compileUnionPlan, scopeContext, type Compiled } from "./build" import { closeAll, targets, type FuzzDialect, type FuzzTarget } from "./dialects" import { FuzzFailure, property, skipKnown, withContext } from "./harness" -import { knownPlanIssue } from "./known-issues" -import { evaluate } from "./model" -import { nastyOptions, plainOptions, plan, type Plan } from "./plan" +import { knownPlanIssue, knownUnionIssue } from "./known-issues" +import { evaluate, evaluateUnion, type ModelResult } from "./model" +import { nastyOptions, plainOptions, plan, unionPlan, type Plan, type UnionPlan } from "./plan" import { normalize } from "./values" afterAll(closeAll) @@ -27,15 +30,33 @@ const canonical = (rows: ReadonlyArray): Array => ) const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort() -const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { +const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - const expected = evaluate(input, target.semantics) - const result = compilePlan(dialect, dialect.context, input) + return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input)) +} + +const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { + const knownUnion = knownUnionIssue({ target: target.name, plan: input }) + if (knownUnion !== undefined) skipKnown(knownUnion) + for (const branch of input.branches) { + const known = knownPlanIssue({ target: target.name, plan: branch }) + if (known !== undefined) skipKnown(known) + } + return checkRows( + target, + input.orgId, + evaluateUnion(input, target.semantics), + compileUnionPlan(dialect, dialect.context, input), + compileUnionPlan(dialect, scopeContext(dialect), input), + ) +} + +/** Run what compiled and compare its rows with the model's; `scoped` is the same input over the plain tables. */ +const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled) => { if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. - const scoped = compilePlan(dialect, scopeContext(dialect), input) const tenantScope = scoped._tag === "Compiled" ? scoped.compiled.tenantScope : undefined const outcome = await target.run(compiled) await withContext({ sql: compiled.sql, parameters: compiled.parameters, tenantScope, outcome, expected: expected.rows }, () => { @@ -46,7 +67,7 @@ const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Pla if (expected.ordered) expect(actual).toEqual(want) else expect(sorted(actual)).toEqual(sorted(want)) if (tenantScope === "single-tenant") { - const foreign = [...expected.orgs].filter((org) => org !== input.orgId) + const foreign = [...expected.orgs].filter((org) => org !== orgId) expect(foreign, "a single-tenant query read another org's rows").toEqual([]) } }) @@ -63,3 +84,15 @@ describe("queries match the model", () => { } } }) + +describe("unions match the model", () => { + for (const { dialect, target } of targets) { + for (const [label, options] of [ + ["plain strings", plainOptions(dialect)], + ["nasty strings", nastyOptions(dialect)], + ] as const) { + const name = `unions match the model (${target.name}, ${label})` + it.skipIf(!target.available)(name, () => property(name, unionPlan(options), (input) => unionMatchesModel(dialect, target, input), { runs: 60 }), 300_000) + } + } +}) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index a21260e..283f51a 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -5,7 +5,7 @@ // target at once). Inputs that hit an open entry in `known-issues.ts` are // skipped until the bug is fixed, then replayed for good. import { DateTime } from "effect" -import type { Plan, RowsShape } from "./plan" +import type { Plan, RowsShape, UnionPlan } from "./plan" const rows = (shape: RowsShape): Plan => ({ orgId: "org_1", join: undefined, where: [], shape, wrap: false, crossTenant: false }) @@ -51,6 +51,20 @@ export const pinned: Record> = { "postgres:text[] select": [[]], "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + // Unions. + "unions match the model": [ + // clickhouse-union-variant-order-by: `Id + 1` (UInt64) under `Id + -1` (Int64), ordered by that column. + { + orgId: "org_1", + branches: [ + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: 1 }]], distinct: false, orderBy: [] }), + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: -1 }]], distinct: false, orderBy: [] }), + ], + orderBy: [["e0", "asc"], ["id", "asc"]], + crossTenant: false, + }, + ] satisfies ReadonlyArray, + // Hostile values. "hostile values are refused, never thrown (clickhouse)": [ { dialect: "clickhouse", column: "OrgId", site: "eq", value: 0n }, diff --git a/tests/fuzz/writes.fuzz.test.ts b/tests/fuzz/writes.fuzz.test.ts new file mode 100644 index 0000000..49ff3ac --- /dev/null +++ b/tests/fuzz/writes.fuzz.test.ts @@ -0,0 +1,351 @@ +// Writes: random sequences of builder writes leave the table the model says. +// +// Each input gets a fresh scratch table on the dialect's first target (see +// `WriteSupport.scratch`), then runs a sequence of writes through the builder: +// multi-row inserts that leave defaulted or nullable columns out, INSERT … +// SELECT, ON CONFLICT DO NOTHING / DO UPDATE (with the existing and the +// proposed row), UPDATE and DELETE with a filter or `allRows()`, and RETURNING. +// After every write the table is read back and compared with a JS model of its +// rows, and RETURNING rows with the rows the model says the write touched. +// +// A write the dialect does not support (`WriteSupport`) is still generated, and +// must be refused at compile time: never run, never thrown. +import fc from "fast-check" +import { afterAll, describe, expect, it } from "vitest" +import { classify } from "./build" +import { closeAll, dialects, type FuzzDialect } from "./dialects" +import type { WriteSupport } from "./dialects/types" +import { FuzzFailure, property, skip, withContext } from "./harness" +import { nastyString, wellFormed } from "./values" + +afterAll(closeAll) + +// Plans + +interface Row { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note: string | null + readonly Qty: number +} +/** A row to insert; `Note` and `Qty` may be left out (NULL, and the database default 5). */ +interface NewRow { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note?: string | null + readonly Qty?: number +} +type QtySet = { readonly k: "value"; readonly v: number } | { readonly k: "add"; readonly v: number } | { readonly k: "excluded" } +interface SetPlan { + readonly Name?: string + readonly Note?: string | null + readonly Qty?: QtySet +} +type Where = + | { readonly k: "id"; readonly op: "eq" | "lt" | "gt"; readonly v: number } + | { readonly k: "name"; readonly v: string } + | { readonly k: "note"; readonly isNull: boolean } + | { readonly k: "org"; readonly v: string } + | { readonly k: "and" | "or"; readonly a: Where; readonly b: Where } +type Conflict = { readonly action: "nothing"; readonly target: boolean } | { readonly action: "update"; readonly set: SetPlan } +type WriteOp = + | { readonly k: "insert"; readonly rows: ReadonlyArray; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "insertSelect"; readonly offset: number; readonly where: Where; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "update"; readonly set: SetPlan; readonly where: Where | "all"; readonly returning: boolean } + | { readonly k: "delete"; readonly where: Where | "all"; readonly returning: boolean } +export interface WritesInput { + readonly ops: ReadonlyArray +} + +const DEFAULT_QTY = 5 +const ids = fc.integer({ min: 1, max: 12 }) +const org = fc.constantFrom("org_1", "org_2") + +const generators = (dialect: FuzzDialect) => { + const text = fc.oneof( + { weight: 3, arbitrary: fc.constantFrom("a", "b", "c", "it's", "x\\y") }, + { weight: 1, arbitrary: nastyString(dialect.strings).filter(wellFormed) }, + ) + const newRow: fc.Arbitrary = fc.record( + { Id: ids, OrgId: org, Name: text, Note: fc.option(text, { nil: null }), Qty: fc.integer({ min: -5, max: 50 }) }, + { requiredKeys: ["Id", "OrgId", "Name"] }, + ) + const qty = (excluded: boolean): fc.Arbitrary => + fc.oneof( + fc.integer({ min: -5, max: 50 }).map((v): QtySet => ({ k: "value", v })), + fc.integer({ min: -3, max: 3 }).map((v): QtySet => ({ k: "add", v })), + ...(excluded ? [fc.constant({ k: "excluded" })] : []), + ) + const set = (excluded: boolean): fc.Arbitrary => + fc + .record({ Name: text, Note: fc.option(text, { nil: null }), Qty: qty(excluded) }, { requiredKeys: [] }) + .filter((s) => Object.keys(s).length > 0) + const leaf: fc.Arbitrary = fc.oneof( + fc.record({ k: fc.constant("id" as const), op: fc.constantFrom("eq" as const, "lt" as const, "gt" as const), v: ids }), + fc.record({ k: fc.constant("name" as const), v: text }), + fc.record({ k: fc.constant("note" as const), isNull: fc.boolean() }), + fc.record({ k: fc.constant("org" as const), v: org }), + ) + const where: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: leaf }, + { weight: 1, arbitrary: fc.record({ k: fc.constantFrom("and" as const, "or" as const), a: leaf, b: leaf }) }, + ) + const conflict: fc.Arbitrary = fc.option( + fc.oneof( + fc.record({ action: fc.constant("nothing" as const), target: fc.boolean() }), + fc.record({ action: fc.constant("update" as const), set: set(true) }), + ), + { nil: undefined, freq: 2 }, + ) + const returning = fc.boolean() + const op: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: fc.record({ k: fc.constant("insert" as const), rows: fc.array(newRow, { minLength: 1, maxLength: 4 }), conflict, returning }, { requiredKeys: ["k", "rows", "returning"] }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("insertSelect" as const), offset: fc.constantFrom(100, 1000, 10_000), where, conflict, returning }, { requiredKeys: ["k", "offset", "where", "returning"] }) }, + { weight: 2, arbitrary: fc.record({ k: fc.constant("update" as const), set: set(false), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("delete" as const), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + ) + // The first write seeds rows, so later ones have something to change. + const seed = fc.array(newRow, { minLength: 1, maxLength: 5 }).map((rows): WriteOp => ({ k: "insert", rows, returning: false })) + return fc.tuple(seed, fc.array(op, { minLength: 1, maxLength: 5 })).map(([first, rest]): WritesInput => ({ ops: [first, ...rest] })) +} + +// The model + +const matches = (where: Where | "all", row: Row): boolean => { + if (where === "all") return true + switch (where.k) { + case "id": + return where.op === "eq" ? row.Id === where.v : where.op === "lt" ? row.Id < where.v : row.Id > where.v + case "name": + return row.Name === where.v + case "note": + return (row.Note === null) === where.isNull + case "org": + return row.OrgId === where.v + case "and": + return matches(where.a, row) && matches(where.b, row) + case "or": + return matches(where.a, row) || matches(where.b, row) + } +} + +const applySet = (set: SetPlan, row: Row, excluded?: Row): Row => ({ + ...row, + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined + ? {} + : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? row.Qty + set.Qty.v : excluded!.Qty }), +}) + +const fullRow = (row: NewRow): Row => ({ Id: row.Id, OrgId: row.OrgId, Name: row.Name, Note: row.Note ?? null, Qty: row.Qty ?? DEFAULT_QTY }) + +/** + * Rows inserted one by one, as the database does: a key that exists is a + * conflict, skipped or updated. Returns the rows the statement inserted or + * updated, which is what RETURNING gives. + */ +const insertRows = (state: Array, rows: ReadonlyArray, conflict: Conflict | undefined): Array => { + const touched: Array = [] + for (const row of rows) { + const at = state.findIndex((r) => r.Id === row.Id) + if (at === -1) { + state.push(row) + touched.push(row) + } else if (conflict?.action === "update") { + state[at] = applySet(conflict.set, state[at]!, row) + touched.push(state[at]!) + } + } + return touched +} + +/** Apply a write to the model; the rows RETURNING gives. */ +const apply = (state: Array, op: WriteOp): Array => { + switch (op.k) { + case "insert": + return insertRows(state, op.rows.map(fullRow), op.conflict) + case "insertSelect": + // The SELECT reads the table as it was before the statement. + return insertRows(state, state.filter((row) => matches(op.where, row)).map((row) => ({ ...row, Id: row.Id + op.offset })), op.conflict) + case "update": { + const touched: Array = [] + for (const [i, row] of state.entries()) { + if (!matches(op.where, row)) continue + state[i] = applySet(op.set, row) + touched.push(state[i]!) + } + return touched + } + case "delete": { + const removed = state.filter((row) => matches(op.where, row)) + const kept = state.filter((row) => !matches(op.where, row)) + state.splice(0, state.length, ...kept) + return removed + } + } +} + +/** + * Make a write valid for the table it meets: an insert without ON CONFLICT + * gets fresh keys where its keys exist (a duplicate key is an error, not a + * finding), and one with DO UPDATE no repeated key (Postgres refuses to update + * a row twice in one statement). + */ +const repair = (state: ReadonlyArray, used: Set, op: WriteOp): WriteOp => { + const fresh = () => { + let id = Math.max(0, ...used) + 1 + while (used.has(id)) id++ + return id + } + if (op.k === "insert") { + const seen = new Set() + const rows = op.rows.map((row) => { + const clash = op.conflict === undefined ? state.some((r) => r.Id === row.Id) || seen.has(row.Id) : op.conflict.action === "update" && seen.has(row.Id) + const id = clash ? fresh() : row.Id + seen.add(id) + used.add(id) + return id === row.Id ? row : { ...row, Id: id } + }) + return { ...op, rows } + } + if (op.k === "insertSelect" && op.conflict === undefined) { + const copies = state.filter((row) => matches(op.where, row)).map((row) => row.Id + op.offset) + if (copies.some((id) => state.some((r) => r.Id === id))) skip("INSERT … SELECT would repeat a key") + for (const id of copies) used.add(id) + } + if (op.k === "insertSelect") for (const row of state) used.add(row.Id + op.offset) + return op +} + +// Capabilities + +/** The first feature `op` uses that the dialect does not support, if any. */ +const unsupported = (support: WriteSupport, op: WriteOp): string | undefined => { + if (op.k === "insertSelect" && !support.insertSelect) return "INSERT … SELECT" + if (op.k === "update" && !support.update) return "UPDATE" + if (op.k === "delete" && !support.delete) return "DELETE" + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined && !support.onConflict) return "ON CONFLICT" + if (op.returning && !support.returning) return "RETURNING" + if (op.k === "insert" && !support.defaultCells && op.rows.length > 1) { + for (const column of ["Note", "Qty"] as const) { + const given = op.rows.filter((row) => Object.hasOwn(row, column)).length + if (given > 0 && given < op.rows.length) return `DEFAULT for a missing ${column} in a multi-row VALUES` + } + } + return undefined +} + +// Builder calls + +const whereCond = ($: any, where: Where): any => { + switch (where.k) { + case "id": + return $.Id[where.op](where.v) + case "name": + return $.Name.eq(where.v) + case "note": + return where.isNull ? $.Note.isNull() : $.Note.isNotNull() + case "org": + return $.OrgId.eq(where.v) + case "and": + case "or": + return whereCond($, where.a)[where.k](whereCond($, where.b)) + } +} + +const setRecord = (set: SetPlan, $: any, excluded?: any): Record => ({ + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined ? {} : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? $.Qty.add(set.Qty.v) : excluded.Qty }), +}) + +const buildWrite = (dialect: FuzzDialect, table: any, op: WriteOp): any => { + const { insertInto, update, deleteFrom, from } = dialect.module + let write: any + const filtered = (w: any, where: Where | "all") => (where === "all" ? w.allRows() : w.where(($: any) => [whereCond($, where)])) + switch (op.k) { + case "insert": + write = insertInto(table).values(op.rows.length === 1 ? (op.rows[0] as never) : (op.rows as never)) + break + case "insertSelect": + write = insertInto(table).select( + (from(table) as any) + .select(($: any) => ({ Id: $.Id.add(op.offset), OrgId: $.OrgId, Name: $.Name, Note: $.Note, Qty: $.Qty })) + .where(($: any) => [whereCond($, op.where)]), + ) + break + case "update": + write = filtered(update(table).set(($: any) => setRecord(op.set, $)), op.where) + break + case "delete": + write = filtered(deleteFrom(table), op.where) + break + } + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined) { + const conflict = op.conflict + write = + conflict.action === "nothing" + ? write.onConflictDoNothing(conflict.target ? { target: ["Id"] } : {}) + : write.onConflictDoUpdate({ target: ["Id"], set: ($: any, excluded: any) => setRecord(conflict.set, $, excluded) }) + } + return op.returning ? write.returning() : write +} + +// The property + +const byId = (rows: ReadonlyArray) => [...rows].sort((a, b) => a.Id - b.Id) +const asRows = (rows: ReadonlyArray): Array => + rows.map((row) => { + const r = row as Record + return { Id: Number(r.Id), OrgId: r.OrgId as string, Name: r.Name as string, Note: (r.Note ?? null) as string | null, Qty: Number(r.Qty) } + }) + +let tables = 0 + +const writesMatchModel = async (dialect: FuzzDialect, input: WritesInput) => { + const { scratch } = dialect.writes + const target = dialect.targets[0]! + const name = `fuzz_writes_${process.pid}_${tables++}` + for (const sql of scratch.create(name)) await scratch.exec(sql) + try { + const table = scratch.table(name) + const read = classify(() => dialect.module.compile((dialect.module.from(table) as any).select(), {})) + if (read._tag !== "Compiled") throw new FuzzFailure("the scratch table's SELECT did not compile", { error: read.error }) + const state: Array = [] + const used = new Set() + for (const [step, planned] of input.ops.entries()) { + const op = repair(state, used, planned) + const missing = unsupported(dialect.writes, op) + const result = classify(() => dialect.module.compile(buildWrite(dialect, table, op), {})) + const context = { step, op, compiled: result._tag === "Compiled" ? { sql: result.compiled.sql, parameters: result.compiled.parameters } : result, before: byId(state) } + if (result._tag === "Threw") throw new FuzzFailure("the builder threw instead of failing", context) + if (missing !== undefined) { + if (result._tag === "Compiled") throw new FuzzFailure(`${missing} compiled for ${dialect.name}, which does not support it`, context) + continue + } + if (result._tag !== "Compiled") throw new FuzzFailure(`a supported write was not compiled (${result._tag})`, context) + const touched = apply(state, op) + const outcome = await target.run(result.compiled) + const after = await target.run(read.compiled) + await withContext({ ...context, outcome, expected: byId(state), returningExpected: op.returning ? byId(touched) : undefined, after }, () => { + expect(outcome._tag).toBe("Rows") + expect(after._tag).toBe("Rows") + if (outcome._tag !== "Rows" || after._tag !== "Rows") return + expect(byId(asRows(after.rows))).toEqual(byId(state)) + if (op.returning) expect(byId(asRows(outcome.rows))).toEqual(byId(touched)) + }) + } + } finally { + await scratch.exec(scratch.drop(name)) + } +} + +describe("writes match the model", () => { + for (const dialect of dialects) { + const name = `writes match the model (${dialect.name})` + it.skipIf(!dialect.targets[0]!.available)(name, () => property(name, generators(dialect), (input) => writesMatchModel(dialect, input), { runs: 40 }), 300_000) + } +}) From 7fcd3cab583656478b98b81b0afd6553fdf77d6a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:09:40 +0100 Subject: [PATCH 2/8] fix(clickhouse): cast a union's mixed numeric column to Float64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A UNION ALL column of numbers whose branches have no common ClickHouse type (`Id + 1` is UInt64 and `Id - 1` Int64; Int64 and Float64) becomes a Variant, and an ORDER BY on it fails ("Variant/Dynamic are not allowed in ORDER BY keys"); with use_variant_as_common_type = 0 the union itself is refused. The union fuzzer found it. On a dialect with `unionNumberCast` (ClickHouse), a column is selected as toFloat64(…) in every branch when every branch selects a number and their types may differ: different declared types, or different computations. Every numeric ClickHouse type decodes to a JS number, so Float64 loses nothing the decode keeps. The known-issues entry is gone; the pinned counterexample now passes. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 8 +++++ docs/unions-and-ctes.md | 10 ++++-- src/ch/compile.test.ts | 20 ++++++++++++ src/ch/compile.ts | 45 +++++++++++++++++++++++--- src/ch/dialect.ts | 9 ++++++ tests/fuzz/known-issues.ts | 22 +------------ 6 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 .changeset/clickhouse-union-numbers.md diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md new file mode 100644 index 0000000..e92afff --- /dev/null +++ b/.changeset/clickhouse-union-numbers.md @@ -0,0 +1,8 @@ +--- +"@maple-dev/effect-orm": patch +--- + +On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and +`Id - 1`, or a `UInt32` and an `Int64` column) is selected as `toFloat64(…)` in each branch. +ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic +are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 8c3f5e2..da64a04 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -28,9 +28,13 @@ themselves keep whatever ordering they declared. Unions compile with **`compileUnion`**, not `compile`. -> **Watch branch types.** ClickHouse unifies column types across `UNION ALL` branches, and a -> branch pairing (say) a `Float64` with a `UInt64` can be rejected outright depending on server -> settings. Cast explicitly in each branch when the types are not already identical. +> **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and +> `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, +> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose +> branches may differ in type (different declared types, or different arithmetic: `Id + 1` is +> `UInt64`, `Id - 1` is `Int64`) is selected as `toFloat64(…)` in every branch. Numbers decode to +> JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical +> expressions are left as they are. Cast other types explicitly when the branches differ. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index e889029..90dd204 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -159,6 +159,26 @@ describe("CompiledQuery.decodeRows", () => { expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] }) }) + it("casts a ClickHouse union's numeric column to one type when its branches may differ", () => { + // `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses. + const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) + const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) + const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) + expect(compiled.sql).toContain("toFloat64(orders.Id + 1) AS shifted") + expect(compiled.sql).toContain("toFloat64(orders.Id + -1) AS shifted") + // One declared type in every branch, and non-numbers, stay as they are. + expect(compiled.sql).not.toContain("toFloat64(orders.Id) AS id") + expect(compiled.sql).not.toContain("toFloat64(orders.Amount)") + expect(compiled.sql).not.toContain("toFloat64(orders.Name)") + // Different declared types do differ. + const mixed = compileUnionUnsafe( + CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), + {}, + ) + expect(mixed.sql).toContain("toFloat64(orders.Id) AS n") + expect(mixed.sql).toContain("toFloat64(orders.Amount) AS n") + }) + it.effect("decodes rows with the declared schema for handwritten SQL", () => Effect.gen(function* () { const compiled = rawCompiledQuery<{ readonly name: string; readonly count: number }>({ diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..8213f78 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -8,7 +8,7 @@ // 4. Assembling into SqlQuery and calling the existing compileQuery() import { compiled, compiledUnsafe, fail } from "./failure" -import { columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" +import { CHFloatResult, CHNumber, columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" import type { CHQuery, CHQueryState, NeedsSelect } from "./query" import type { CHUnionQuery } from "./union" import { isInsert, type AnyInsert, type CHInsert } from "./insert" @@ -22,7 +22,7 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { track, untracked, type RenderTrack } from "../sql/render-tracker" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param" -import { mergeResultSchemas } from "./define-fn" +import { mergeResultSchemas, schemaOf } from "./define-fn" import { encodeValue } from "./literal" import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" import { Effect, Option, Schema } from "effect" @@ -718,6 +718,8 @@ function compileInner< params: Params, options?: { selectKeys?: ReadonlyArray + /** Union columns cast with the dialect's `unionNumberCast`; see `unionNumberCasts`. */ + castNumbers?: ReadonlySet skipFormat?: boolean rowSchema?: CompiledQueryRowSchema /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. @@ -922,7 +924,8 @@ function compileInner< } const fragment = expr.toFragment() const [sql, found] = track(() => compileSqlFragment(fragment)) - return { alias, sql: castSelectedParam(expr, sql), found, column: plainColumn(fragment) } + const numberCast = options?.castNumbers?.has(alias) === true ? currentDialect().unionNumberCast : undefined + return { alias, sql: numberCast !== undefined ? numberCast(castSelectedParam(expr, sql)) : castSelectedParam(expr, sql), found, column: plainColumn(fragment) } }) const where = presentWhere.map((c) => raw(rowFilter("WHERE", c))) const having = (state.havingFn ? state.havingFn($) : []) @@ -1391,6 +1394,39 @@ export function compileUnionRaw>( } /** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */ +/** A JS-number schema: every numeric ClickHouse type and arithmetic result decodes through one of these. */ +const isNumberSchema = (schema: unknown): boolean => { + if (schema === CHNumber || schema === CHFloatResult) return true + // NullOr(CHNumber): a nullable column or arithmetic over one. + const members = (schema as { members?: ReadonlyArray } | undefined)?.members + return Array.isArray(members) && members.some((m) => m === CHNumber || m === CHFloatResult) && members.every((m) => m === CHNumber || m === CHFloatResult || m === Schema.Null) +} + +/** + * The union columns to cast to one numeric type, on a dialect with a + * `unionNumberCast`: every branch selects a number there, and their types may + * differ, because the branches declare different column types, or compute it + * differently (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). + */ +function unionNumberCasts(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { + const casts = new Set() + if (currentDialect().unionNumberCast === undefined || queries.length < 2) return casts + for (const alias of selectKeys) { + const exprs = queries.map((q) => (selectExprsOf(q) as Record | undefined)?.[alias]) + const kinds = exprs.map((expr) => { + if (!isExprLike(expr)) return undefined + const declared = columnTypeOf(expr as Expr) + if (declared !== undefined) return isNumberSchema(declared.schema) ? declared.sql : undefined + return isNumberSchema(schemaOf(expr)) ? "computed" : undefined + }) + if (kinds.some((kind) => kind === undefined)) continue + if (new Set(kinds).size > 1) casts.add(alias) + // The same computation in every branch (a `sum` over one column) has one type; different ones may not. + else if (kinds[0] === "computed" && new Set(exprs.map((expr) => compileSqlFragment((expr as Expr).toFragment()))).size > 1) casts.add(alias) + } + return casts +} + function compileUnionInner, Params extends Record>( union: CHUnionQuery, params: Params, @@ -1416,8 +1452,9 @@ function compileUnionInner, Params extends Re if (state.queries.some((q) => q._state.lock !== undefined)) { return fail(new QueryBuilderDefect({ message: "unionAll: a branch cannot take a row lock; lock in a query over the union instead" }),failedQuery()) } + const castNumbers = unionNumberCasts(state.queries, selectKeys) const subQueries = state.queries.map((q) => - compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), + compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes, castNumbers }), ) const bounds = new Set( subQueries.flatMap((q) => { diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b6c97..d77dee7 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -151,6 +151,13 @@ export interface Dialect extends Omit { * unchecked: a custom dialect says which set it renders, if either. */ readonly functions?: string + /** + * Casts a numeric UNION ALL column whose branches may have different types + * to one type. ClickHouse has no common type for `UInt64` and `Int64` (or + * `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY + * refuses. Absent: the dialect unifies numeric branches itself. + */ + readonly unionNumberCast?: (sql: string) => string } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -173,6 +180,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", + // Every numeric ClickHouse type decodes to a JS number (CHNumber), so Float64 loses nothing the decode keeps. + unionNumberCast: (sql) => `toFloat64(${sql})`, } // The dialect of the enclosing compile, beside the syntax installed for the diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index 0bf51b9..e487af7 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -102,27 +102,7 @@ export interface UnionInput { const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) const unionIssues: ReadonlyArray> = [ - { - id: "clickhouse-union-variant-order-by", - reason: - "On ClickHouse 26, a UNION ALL column whose branches have numeric types with no common supertype (`Id + 1` is " + - "UInt64, `Id - 1` Int64; Int64 and Float64) becomes a Variant, and the union's ORDER BY on it fails with " + - "`Data types Variant/Dynamic are not allowed in ORDER BY keys`. Fix: have compileUnion cast a numeric union " + - "column to one type (e.g. Float64, or the widest signed type) when its branches' expressions differ.", - matches: ({ target, plan }) => - target.startsWith("clickhouse") && - plan.orderBy.some(([alias]) => { - const kinds = new Set( - plan.branches.map((branch) => { - const e = selected(branch, alias) - return e === undefined || (e.k !== "num" && e.k !== "arith" && !(e.k === "col" && (e.col === "Id" || e.col === "Amount"))) - ? "other" - : `${chKind(e)}:${e.k === "col" ? e.col : e.k}` - }), - ) - return kinds.size > 1 && !kinds.has("other") - }), - }, + ] export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) From 79d0d722dc820e86f8c8927e062f139bbc2e63e9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:17:35 +0100 Subject: [PATCH 3/8] test(fuzz): a query pinned to one org by its own filter is single-tenant for that org The tenant oracle took "single-tenant" to mean the orgId param's tenant. `OrgId IN ('org_2')` pins a query to org_2, which the builder rightly reports as single-tenant; the oracle called reading org_2 a leak. Where the plan's WHERE filters OrgId itself, any one org is right; otherwise it must still be the param's. Found by a 20x soak on D1; pinned. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/queries.fuzz.test.ts | 10 +++++++--- tests/fuzz/regressions.ts | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index c836dd8..890cc31 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -33,7 +33,7 @@ const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input)) + return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input), input.where) } const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { @@ -49,11 +49,12 @@ const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Unio evaluateUnion(input, target.semantics), compileUnionPlan(dialect, dialect.context, input), compileUnionPlan(dialect, scopeContext(dialect), input), + input.branches.map((branch) => branch.where), ) } /** Run what compiled and compare its rows with the model's; `scoped` is the same input over the plain tables. */ -const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled) => { +const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled, where: unknown) => { if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. @@ -67,7 +68,10 @@ const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResul if (expected.ordered) expect(actual).toEqual(want) else expect(sorted(actual)).toEqual(sorted(want)) if (tenantScope === "single-tenant") { - const foreign = [...expected.orgs].filter((org) => org !== orgId) + // The tenant is the `orgId` param, unless the plan also constrains OrgId itself (`OrgId IN ('org_2')` + // pins the query to org_2): then it is single-tenant by that, and any one org is right. + const pinnedElsewhere = /"col":"OrgId"/.test(JSON.stringify(where)) + const foreign = pinnedElsewhere ? [...expected.orgs].slice(1) : [...expected.orgs].filter((org) => org !== orgId) expect(foreign, "a single-tenant query read another org's rows").toEqual([]) } }) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 283f51a..8193925 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -36,6 +36,11 @@ export const pinned: Record> = { { k: "or", style: "method", cs: [{ k: "and", style: "method", cs: [{ k: "const", v: true }, { k: "tenant", side: "orders" }] }, { k: "tenant", side: "orders" }] }, ], }, + // `OrgId IN ('org_2')` pins the query to org_2, not to the orgId param: single-tenant, and right to read org_2. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + where: [{ k: "in", neg: false, e: { k: "col", col: "OrgId" }, values: ["org_2"] }], + }, // pg-fraction-bound-to-integer. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), From 4a7f6d3700a2987c827aa88bfb485cdb7ca58837 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:26:04 +0100 Subject: [PATCH 4/8] fix(clickhouse): cast a union's numeric column around each branch, not inside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inside a branch the cast ran before that branch's DISTINCT, so two UInt64 values past 2^53 (one Float64) became one row. The cast now wraps each branch, SELECT toFloat64(x) AS x, … FROM (), so DISTINCT, GROUP BY and LIMIT see exact values (CodeRabbit). The writes suite now accepts only a QueryBuilderError or a QueryBuilderDefect as the refusal of a clause a dialect lacks, not any defect. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 2 +- docs/unions-and-ctes.md | 5 +++-- src/ch/compile.test.ts | 14 +++++++------- src/ch/compile.ts | 18 +++++++++++------- tests/fuzz/writes.fuzz.test.ts | 4 ++++ 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index e92afff..0296930 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -3,6 +3,6 @@ --- On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and -`Id - 1`, or a `UInt32` and an `Int64` column) is selected as `toFloat64(…)` in each branch. +`Id - 1`, or a `UInt32` and an `Int64` column) is cast with `toFloat64(…)` around each branch. ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index da64a04..54f68cb 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -32,8 +32,9 @@ Unions compile with **`compileUnion`**, not `compile`. > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, > with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose > branches may differ in type (different declared types, or different arithmetic: `Id + 1` is -> `UInt64`, `Id - 1` is `Int64`) is selected as `toFloat64(…)` in every branch. Numbers decode to -> JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical +> `UInt64`, `Id - 1` is `Int64`) is cast with `toFloat64(…)` around every branch +> (`SELECT toFloat64(x) AS x, … FROM ()`), so a branch's own DISTINCT, GROUP BY and LIMIT +> still see exact values. Numbers decode to JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical > expressions are left as they are. Cast other types explicitly when the branches differ. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 90dd204..3cc4f6f 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -164,19 +164,19 @@ describe("CompiledQuery.decodeRows", () => { const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) - expect(compiled.sql).toContain("toFloat64(orders.Id + 1) AS shifted") - expect(compiled.sql).toContain("toFloat64(orders.Id + -1) AS shifted") + // Around each branch, so a branch's own DISTINCT sees the exact values. + expect(compiled.sql.match(/SELECT id, toFloat64\(shifted\) AS shifted, total, name FROM \(/g)).toHaveLength(2) + expect(compiled.sql).toContain("orders.Id + 1 AS shifted") // One declared type in every branch, and non-numbers, stay as they are. - expect(compiled.sql).not.toContain("toFloat64(orders.Id) AS id") - expect(compiled.sql).not.toContain("toFloat64(orders.Amount)") - expect(compiled.sql).not.toContain("toFloat64(orders.Name)") + expect(compiled.sql).not.toContain("toFloat64(id)") + expect(compiled.sql).not.toContain("toFloat64(total)") + expect(compiled.sql).not.toContain("toFloat64(name)") // Different declared types do differ. const mixed = compileUnionUnsafe( CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), {}, ) - expect(mixed.sql).toContain("toFloat64(orders.Id) AS n") - expect(mixed.sql).toContain("toFloat64(orders.Amount) AS n") + expect(mixed.sql.match(/SELECT toFloat64\(n\) AS n FROM \(/g)).toHaveLength(2) }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 8213f78..d69b875 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -718,8 +718,6 @@ function compileInner< params: Params, options?: { selectKeys?: ReadonlyArray - /** Union columns cast with the dialect's `unionNumberCast`; see `unionNumberCasts`. */ - castNumbers?: ReadonlySet skipFormat?: boolean rowSchema?: CompiledQueryRowSchema /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. @@ -924,8 +922,7 @@ function compileInner< } const fragment = expr.toFragment() const [sql, found] = track(() => compileSqlFragment(fragment)) - const numberCast = options?.castNumbers?.has(alias) === true ? currentDialect().unionNumberCast : undefined - return { alias, sql: numberCast !== undefined ? numberCast(castSelectedParam(expr, sql)) : castSelectedParam(expr, sql), found, column: plainColumn(fragment) } + return { alias, sql: castSelectedParam(expr, sql), found, column: plainColumn(fragment) } }) const where = presentWhere.map((c) => raw(rowFilter("WHERE", c))) const having = (state.havingFn ? state.havingFn($) : []) @@ -1452,10 +1449,17 @@ function compileUnionInner, Params extends Re if (state.queries.some((q) => q._state.lock !== undefined)) { return fail(new QueryBuilderDefect({ message: "unionAll: a branch cannot take a row lock; lock in a query over the union instead" }),failedQuery()) } - const castNumbers = unionNumberCasts(state.queries, selectKeys) const subQueries = state.queries.map((q) => - compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes, castNumbers }), + compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) + // Cast around each branch, not inside it: a branch's DISTINCT, GROUP BY and LIMIT still see the + // exact values (two UInt64s past 2^53 are one Float64). + const castNumbers = unionNumberCasts(state.queries, selectKeys) + const numberCast = currentDialect().unionNumberCast + const branchSql = (sql: string): string => + castNumbers.size === 0 || numberCast === undefined + ? sql + : `SELECT ${selectKeys.map((key) => (castNumbers.has(key) ? `${numberCast(quoteIdent(key))} AS ${quoteIdent(key)}` : quoteIdent(key))).join(", ")} FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__branch")}` : ""}` const bounds = new Set( subQueries.flatMap((q) => { const bound = tenantBoundOf(q) @@ -1479,7 +1483,7 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => branchSql(q.sql)).map((q) => (parenthesize ? `(\n${q}\n)` : q)).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = diff --git a/tests/fuzz/writes.fuzz.test.ts b/tests/fuzz/writes.fuzz.test.ts index 49ff3ac..409cad9 100644 --- a/tests/fuzz/writes.fuzz.test.ts +++ b/tests/fuzz/writes.fuzz.test.ts @@ -324,6 +324,10 @@ const writesMatchModel = async (dialect: FuzzDialect, input: WritesInput) => { if (result._tag === "Threw") throw new FuzzFailure("the builder threw instead of failing", context) if (missing !== undefined) { if (result._tag === "Compiled") throw new FuzzFailure(`${missing} compiled for ${dialect.name}, which does not support it`, context) + // A clause the dialect lacks fails with a typed error, or dies with a QueryBuilderDefect (src/ch/errors.ts); nothing else. + if (result._tag !== "Refused" && !(result._tag === "Defect" && result.error.includes("QueryBuilderDefect"))) { + throw new FuzzFailure(`${missing} was not refused with a QueryBuilderError or QueryBuilderDefect`, context) + } continue } if (result._tag !== "Compiled") throw new FuzzFailure(`a supported write was not compiled (${result._tag})`, context) From b0619ae0c8de65c370dc1cc5db32266dba843060 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:30:45 +0100 Subject: [PATCH 5/8] fix(clickhouse): order a union's mixed numeric column by exact keys, never cast its values The previous fix cast such a column to Float64, which an independent review showed corrupts integers past 2^53 wherever the values stay on the server: INSERT ... SELECT from the union stored 9007199254740992 for 9007199254740993, and an IN over it matched nothing. Only the union's ORDER BY was ever broken (a Variant is not a valid sort key), so only the sort key changes now: toFloat64(x), then accurateCastOrNull(x, 'Int128'), which keeps integers past 2^53 in exact order. Checked on ClickHouse 26.8 with UInt64/Int64 values around 2^53 and 2^64 - 1. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 9 ++++---- docs/unions-and-ctes.md | 13 ++++++----- src/ch/compile.test.ts | 20 +++++++--------- src/ch/compile.ts | 32 +++++++++++++------------- src/ch/dialect.ts | 16 +++++++------ 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index 0296930..cb2facb 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -2,7 +2,8 @@ "@maple-dev/effect-orm": patch --- -On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and -`Id - 1`, or a `UInt32` and an `Int64` column) is cast with `toFloat64(…)` around each branch. -ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic -are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. +On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1` +and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic +are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by +`toFloat64(x), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the +values themselves are not converted. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 54f68cb..53c08c1 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -30,12 +30,13 @@ Unions compile with **`compileUnion`**, not `compile`. > **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, -> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose -> branches may differ in type (different declared types, or different arithmetic: `Id + 1` is -> `UInt64`, `Id - 1` is `Int64`) is cast with `toFloat64(…)` around every branch -> (`SELECT toFloat64(x) AS x, … FROM ()`), so a branch's own DISTINCT, GROUP BY and LIMIT -> still see exact values. Numbers decode to JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical -> expressions are left as they are. Cast other types explicitly when the branches differ. +> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy` +> sorts a numeric column whose branches may differ in type (different declared types, or different +> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `toFloat64(x)` and then +> `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values +> themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them +> exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that +> needs one type, should cast it in each branch. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 3cc4f6f..50e1c99 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -159,24 +159,20 @@ describe("CompiledQuery.decodeRows", () => { expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] }) }) - it("casts a ClickHouse union's numeric column to one type when its branches may differ", () => { + it("orders a ClickHouse union's mixed numeric column by exact keys, leaving its values alone", () => { // `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses. const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) - const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) - // Around each branch, so a branch's own DISTINCT sees the exact values. - expect(compiled.sql.match(/SELECT id, toFloat64\(shifted\) AS shifted, total, name FROM \(/g)).toHaveLength(2) - expect(compiled.sql).toContain("orders.Id + 1 AS shifted") - // One declared type in every branch, and non-numbers, stay as they are. - expect(compiled.sql).not.toContain("toFloat64(id)") - expect(compiled.sql).not.toContain("toFloat64(total)") - expect(compiled.sql).not.toContain("toFloat64(name)") - // Different declared types do differ. + const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {}) + // The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact. + expect(compiled.sql).not.toContain("toFloat64(orders") + expect(compiled.sql).toContain("ORDER BY toFloat64(shifted) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") + // Different declared types are mixed too; one declared type in every branch is not. const mixed = compileUnionUnsafe( - CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), + CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]), {}, ) - expect(mixed.sql.match(/SELECT toFloat64\(n\) AS n FROM \(/g)).toHaveLength(2) + expect(mixed.sql).toContain("ORDER BY toFloat64(n) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/compile.ts b/src/ch/compile.ts index d69b875..afb784b 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -1400,14 +1400,14 @@ const isNumberSchema = (schema: unknown): boolean => { } /** - * The union columns to cast to one numeric type, on a dialect with a - * `unionNumberCast`: every branch selects a number there, and their types may - * differ, because the branches declare different column types, or compute it - * differently (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). + * The union columns a dialect with `unionNumberOrderKeys` orders through those + * keys: every branch selects a number, and their types may differ, because the + * branches declare different column types or compute it differently + * (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). */ -function unionNumberCasts(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { +function mixedNumberColumns(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { const casts = new Set() - if (currentDialect().unionNumberCast === undefined || queries.length < 2) return casts + if (currentDialect().unionNumberOrderKeys === undefined || queries.length < 2) return casts for (const alias of selectKeys) { const exprs = queries.map((q) => (selectExprsOf(q) as Record | undefined)?.[alias]) const kinds = exprs.map((expr) => { @@ -1452,14 +1452,7 @@ function compileUnionInner, Params extends Re const subQueries = state.queries.map((q) => compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) - // Cast around each branch, not inside it: a branch's DISTINCT, GROUP BY and LIMIT still see the - // exact values (two UInt64s past 2^53 are one Float64). - const castNumbers = unionNumberCasts(state.queries, selectKeys) - const numberCast = currentDialect().unionNumberCast - const branchSql = (sql: string): string => - castNumbers.size === 0 || numberCast === undefined - ? sql - : `SELECT ${selectKeys.map((key) => (castNumbers.has(key) ? `${numberCast(quoteIdent(key))} AS ${quoteIdent(key)}` : quoteIdent(key))).join(", ")} FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__branch")}` : ""}` + const mixedNumbers = mixedNumberColumns(state.queries, selectKeys) const bounds = new Set( subQueries.flatMap((q) => { const bound = tenantBoundOf(q) @@ -1483,7 +1476,7 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => branchSql(q.sql)).map((q) => (parenthesize ? `(\n${q}\n)` : q)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = @@ -1492,7 +1485,14 @@ function compileUnionInner, Params extends Re if (hasOuter) { sql = `SELECT * FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__union")}` : ""}` if (state.outerOrderBySpecs.length > 0) { - sql += `\nORDER BY ${orderByClause(state.outerOrderBySpecs).join(", ")}` + const orderKeys = currentDialect().unionNumberOrderKeys + const clause = orderByClause(state.outerOrderBySpecs).map((rendered, i) => { + const [column, direction] = state.outerOrderBySpecs[i]! + return orderKeys !== undefined && mixedNumbers.has(column) + ? orderKeys(quoteIdent(column)).map((key) => `${key} ${direction.toUpperCase()}`).join(", ") + : rendered + }) + sql += `\nORDER BY ${clause.join(", ")}` } if (state.outerLimitValue != null) { sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index d77dee7..9a8d7b9 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -152,12 +152,14 @@ export interface Dialect extends Omit { */ readonly functions?: string /** - * Casts a numeric UNION ALL column whose branches may have different types - * to one type. ClickHouse has no common type for `UInt64` and `Int64` (or - * `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY - * refuses. Absent: the dialect unifies numeric branches itself. + * The ORDER BY keys for a numeric UNION ALL column whose branches may have + * different types. ClickHouse has no common type for `UInt64` and `Int64` + * (or `Int64` and `Float64`): the column is a `Variant`, which ORDER BY + * refuses as a key. The values are left as they are (an INSERT ... SELECT or + * an IN over the union keeps them exact); only the sort key is converted. + * Absent: the dialect orders such a column as it is. */ - readonly unionNumberCast?: (sql: string) => string + readonly unionNumberOrderKeys?: (column: string) => ReadonlyArray } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -180,8 +182,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", - // Every numeric ClickHouse type decodes to a JS number (CHNumber), so Float64 loses nothing the decode keeps. - unionNumberCast: (sql) => `toFloat64(${sql})`, + // Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. + unionNumberOrderKeys: (column) => [`toFloat64(${column})`, `accurateCastOrNull(${column}, 'Int128')`], } // The dialect of the enclosing compile, beside the syntax installed for the From 6ca87b00d337589bb027f822a1d6dcc639efd919 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:46:47 +0100 Subject: [PATCH 6/8] fix(clickhouse): order union numbers with CAST, which ClickHouse 26.2 accepts on a Variant toFloat64 refuses a Variant on ClickHouse 26.2 (CI); CAST(x AS Nullable(Float64)) works on 26.2 and 26.8, Variant or not. Checked both versions with the full suite and every fuzz property at 20x. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 2 +- docs/unions-and-ctes.md | 2 +- src/ch/compile.test.ts | 4 ++-- src/ch/dialect.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index cb2facb..dc4ec1e 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -5,5 +5,5 @@ On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1` and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by -`toFloat64(x), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the +`CAST(x AS Nullable(Float64)), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the values themselves are not converted. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 53c08c1..4d591fc 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -32,7 +32,7 @@ Unions compile with **`compileUnion`**, not `compile`. > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, > with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy` > sorts a numeric column whose branches may differ in type (different declared types, or different -> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `toFloat64(x)` and then +> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `CAST(x AS Nullable(Float64))` and then > `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values > themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them > exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 50e1c99..d67fe58 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -166,13 +166,13 @@ describe("CompiledQuery.decodeRows", () => { const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {}) // The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact. expect(compiled.sql).not.toContain("toFloat64(orders") - expect(compiled.sql).toContain("ORDER BY toFloat64(shifted) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") + expect(compiled.sql).toContain("ORDER BY CAST(shifted AS Nullable(Float64)) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") // Different declared types are mixed too; one declared type in every branch is not. const mixed = compileUnionUnsafe( CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]), {}, ) - expect(mixed.sql).toContain("ORDER BY toFloat64(n) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") + expect(mixed.sql).toContain("ORDER BY CAST(n AS Nullable(Float64)) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 9a8d7b9..07ec134 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -182,8 +182,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", - // Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. - unionNumberOrderKeys: (column) => [`toFloat64(${column})`, `accurateCastOrNull(${column}, 'Int128')`], + // CAST, not toFloat64: ClickHouse 26.2's toFloat64 refuses a Variant. Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. + unionNumberOrderKeys: (column) => [`CAST(${column} AS Nullable(Float64))`, `accurateCastOrNull(${column}, 'Int128')`], } // The dialect of the enclosing compile, beside the syntax installed for the From cfeaf59c55e143f93b434dd7f0dab99b6c40519c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:55:46 +0100 Subject: [PATCH 7/8] test(fuzz): rerun ClickHouse's one-argument and() bug without the rewrite, rows still checked The known-issue matcher for this ClickHouse server bug covered two shapes; a 20x soak on 26.2 found a third (`(a OR 0 OR a) AND (a OR 0)`). Rather than chase shapes, a ClickHouse target that gets exactly this error runs the query again with optimize_extract_common_expressions = 0, and its rows are compared with the model as usual: these inputs are now checked, not skipped. The matcher and its helpers are gone. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/dialects/clickhouse.ts | 11 +++++++- tests/fuzz/known-issues.ts | 47 ++----------------------------- tests/fuzz/regressions.ts | 2 +- 3 files changed, 13 insertions(+), 47 deletions(-) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index f130294..a93962b 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -37,8 +37,17 @@ export const clickhouseText = async ( } } +/** + * A ClickHouse server bug, not the builder's: the common-expression rewrite can + * leave a one-argument and() (`a AND ((b AND a) OR a)`, `(a OR 0 OR a) AND (a OR 0)`, + * and more shapes than a matcher keeps up with), which fails with this error. + * The query is run again with the rewrite off, and its rows still checked. + */ +const SERVER_AND_BUG = /Number of arguments for function "and" should be at least 2/ + const runClickHouse = async (compiled: CH.CompiledQuery, settings: Record): Promise => { - const { ok, text } = await clickhouseText(compiled.sql, settings) + let { ok, text } = await clickhouseText(compiled.sql, settings) + if (!ok && SERVER_AND_BUG.test(text)) ({ ok, text } = await clickhouseText(compiled.sql, { ...settings, optimize_extract_common_expressions: "0" })) if (!ok) return { _tag: "DatabaseError", message: text.trim() } const wire = text .split("\n") diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index e487af7..e47e536 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -41,7 +41,6 @@ export const knownHostileIssue = (input: HostileInput): string | undefined => // Query plans (queries.fuzz.test.ts) -import { chKind } from "./model" import type { Plan, UnionPlan } from "./plan" export interface PlanInput { @@ -50,44 +49,7 @@ export interface PlanInput { readonly plan: Plan } -/** Every object in a plan, depth-first: plans are plain data, so this reaches every expression. */ -const nodes = function* (value: unknown): Generator> { - if (Array.isArray(value)) for (const item of value) yield* nodes(item) - else if (value !== null && typeof value === "object") { - yield value as Record - for (const item of Object.values(value)) yield* nodes(item) - } -} - -/** The conjuncts a WHERE list ANDs together, flattening nested ANDs. */ -const conjuncts = (conds: ReadonlyArray): Array => - conds.flatMap((c: any) => (c?.k === "and" ? conjuncts(c.cs) : [c])) -const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) - -const planIssues: ReadonlyArray> = [ - { - id: "clickhouse-server-and-absorption", - reason: - "A ClickHouse 25.8 server bug, not the builder: `WHERE a AND ((b AND a) OR a)` or `a AND ((a OR a) OR a)` fails with " + - "`Number of arguments for function \"and\" should be at least 2` — the common-expression rewrite leaves a " + - "one-argument and(). `SETTINGS optimize_extract_common_expressions = 0` avoids it.", - matches: ({ target, plan }) => { - if (!target.startsWith("clickhouse")) return false - const top = conjuncts(plan.where) - const disjuncts = (c: any): Array => (c?.k === "or" ? c.cs.flatMap(disjuncts) : [c]) - // Fails when every disjunct of a nested OR contains a top-level conjunct `w` (as `w`, or inside an AND): - // `w AND ((x AND w) OR w)`, `w AND ((w OR w) OR w)`. `w AND (w OR w)` and `w AND ((x AND w) OR y)` run. - return [...nodes(plan)].some( - (node) => - node.k === "or" && - (node.cs as Array).some((d) => d?.k === "or" || d?.k === "and") && - top.some((w) => disjuncts(node).every((d) => same(d, w) || (d?.k === "and" && conjuncts(d.cs).some((x) => same(x, w))))), - ) - }, - }, - - -] +const planIssues: ReadonlyArray> = [] export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) @@ -98,11 +60,6 @@ export interface UnionInput { readonly plan: UnionPlan } -/** The expression a branch selects under `alias`. */ -const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) - -const unionIssues: ReadonlyArray> = [ - -] +const unionIssues: ReadonlyArray> = [] export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 8193925..05e77f2 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -28,7 +28,7 @@ export const pinned: Record> = { }), // fromquery-select-all. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, - // clickhouse-server-and-absorption. + // A ClickHouse server bug (a one-argument and()): run again without the rewrite, rows still checked (dialects/clickhouse.ts). { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), where: [ From 433213658d8c3a8014ab523d74e94c581399086b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:57:06 +0100 Subject: [PATCH 8/8] test(fuzz): pin the and() server-bug shape ClickHouse 26.2 hit Fails with TOO_FEW_ARGUMENTS_FOR_FUNCTION if the rerun without the rewrite is removed; passes, rows checked, with it. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/regressions.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 05e77f2..5f0b92d 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -28,6 +28,21 @@ export const pinned: Record> = { }), // fromquery-select-all. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, + // The same server bug in a shape the old matcher missed (ClickHouse 26.2): `(a OR 0 OR a) AND (a OR 0)`. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + orgId: "org_3", + where: [ + { + k: "and", + style: "method", + cs: [ + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "const", v: false }, { k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }] }, + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "in", neg: false, e: { k: "str", v: "" }, values: [] }] }, + ], + }, + ], + }, // A ClickHouse server bug (a one-argument and()): run again without the rewrite, rows still checked (dialects/clickhouse.ts). { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }),