test(fuzz): fuzz unions and writes on every dialect

.changeset/clickhouse-union-numbers.md

@@ -0,0 +1,9 @@
+---
+"@maple-dev/effect-orm": patch
+---
+
+On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1`
+and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic
+are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by
+`CAST(x AS Nullable(Float64)), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the
+values themselves are not converted.

docs/unions-and-ctes.md

@@ -28,9 +28,15 @@ themselves keep whatever ordering they declared.
 
 Unions compile with **`compileUnion`**, not `compile`.
 
-> **Watch branch types.** ClickHouse unifies column types across `UNION ALL` branches, and a
-> branch pairing (say) a `Float64` with a `UInt64` can be rejected outright depending on server
-> settings. Cast explicitly in each branch when the types are not already identical.
+> **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and
+> `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or,
+> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy`
+> sorts a numeric column whose branches may differ in type (different declared types, or different
+> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `CAST(x AS Nullable(Float64))` and then
+> `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values
+> themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them
+> exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that
+> needs one type, should cast it in each branch.
 
 _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_
 

src/ch/compile.test.ts

@@ -159,6 +159,22 @@ describe("CompiledQuery.decodeRows", () => {
 		expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] })
 	})
 
+	it("orders a ClickHouse union's mixed numeric column by exact keys, leaving its values alone", () => {
+		// `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses.
+		const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string })
+		const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name }))
+		const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {})
+		// The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact.
+		expect(compiled.sql).not.toContain("toFloat64(orders")
+		expect(compiled.sql).toContain("ORDER BY CAST(shifted AS Nullable(Float64)) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC")
+		// Different declared types are mixed too; one declared type in every branch is not.
+		const mixed = compileUnionUnsafe(
+			CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]),
+			{},
+		)
+		expect(mixed.sql).toContain("ORDER BY CAST(n AS Nullable(Float64)) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC")
+	})
+
 	it.effect("decodes rows with the declared schema for handwritten SQL", () =>
 		Effect.gen(function* () {
 			const compiled = rawCompiledQuery<{ readonly name: string; readonly count: number }>({

src/ch/compile.ts

@@ -8,7 +8,7 @@
 // 4. Assembling into SqlQuery and calling the existing compileQuery()
 
 import { compiled, compiledUnsafe, fail } from "./failure"
-import { columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types"
+import { CHFloatResult, CHNumber, columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types"
 import type { CHQuery, CHQueryState, NeedsSelect } from "./query"
 import type { CHUnionQuery } from "./union"
 import { isInsert, type AnyInsert, type CHInsert } from "./insert"
@@ -22,7 +22,7 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder"
 import { compileQuery, type SqlQuery } from "../sql/sql-query"
 import { track, untracked, type RenderTrack } from "../sql/render-tracker"
 import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param"
-import { mergeResultSchemas } from "./define-fn"
+import { mergeResultSchemas, schemaOf } from "./define-fn"
 import { encodeValue } from "./literal"
 import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect"
 import { Effect, Option, Schema } from "effect"
@@ -1391,6 +1391,39 @@ export function compileUnionRaw<Output extends Record<string, any>>(
 }
 
 /** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */
+/** A JS-number schema: every numeric ClickHouse type and arithmetic result decodes through one of these. */
+const isNumberSchema = (schema: unknown): boolean => {
+	if (schema === CHNumber || schema === CHFloatResult) return true
+	// NullOr(CHNumber): a nullable column or arithmetic over one.
+	const members = (schema as { members?: ReadonlyArray<unknown> } | undefined)?.members
+	return Array.isArray(members) && members.some((m) => m === CHNumber || m === CHFloatResult) && members.every((m) => m === CHNumber || m === CHFloatResult || m === Schema.Null)
+}
+
+/**
+ * The union columns a dialect with `unionNumberOrderKeys` orders through those
+ * keys: every branch selects a number, and their types may differ, because the
+ * branches declare different column types or compute it differently
+ * (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64).
+ */
+function mixedNumberColumns(queries: ReadonlyArray<CHQuery<any, any, any, any>>, selectKeys: ReadonlyArray<string>): ReadonlySet<string> {
+	const casts = new Set<string>()
+	if (currentDialect().unionNumberOrderKeys === undefined || queries.length < 2) return casts
+	for (const alias of selectKeys) {
+		const exprs = queries.map((q) => (selectExprsOf(q) as Record<string, unknown> | undefined)?.[alias])
+		const kinds = exprs.map((expr) => {
+			if (!isExprLike(expr)) return undefined
+			const declared = columnTypeOf(expr as Expr<any>)
+			if (declared !== undefined) return isNumberSchema(declared.schema) ? declared.sql : undefined
+			return isNumberSchema(schemaOf(expr)) ? "computed" : undefined
+		})
+		if (kinds.some((kind) => kind === undefined)) continue
+		if (new Set(kinds).size > 1) casts.add(alias)
+		// The same computation in every branch (a `sum` over one column) has one type; different ones may not.
+		else if (kinds[0] === "computed" && new Set(exprs.map((expr) => compileSqlFragment((expr as Expr<any>).toFragment()))).size > 1) casts.add(alias)
+	}
+	return casts
+}
+
 function compileUnionInner<Output extends Record<string, any>, Params extends Record<string, any>>(
 	union: CHUnionQuery<Output>,
 	params: Params,
@@ -1419,6 +1452,7 @@ function compileUnionInner<Output extends Record<string, any>, Params extends Re
 	const subQueries = state.queries.map((q) =>
 		compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }),
 	)
+	const mixedNumbers = mixedNumberColumns(state.queries, selectKeys)
 	const bounds = new Set(
 		subQueries.flatMap((q) => {
 			const bound = tenantBoundOf(q)
@@ -1451,7 +1485,14 @@ function compileUnionInner<Output extends Record<string, any>, Params extends Re
 	if (hasOuter) {
 		sql = `SELECT * FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__union")}` : ""}`
 		if (state.outerOrderBySpecs.length > 0) {
-			sql += `\nORDER BY ${orderByClause(state.outerOrderBySpecs).join(", ")}`
+			const orderKeys = currentDialect().unionNumberOrderKeys
+			const clause = orderByClause(state.outerOrderBySpecs).map((rendered, i) => {
+				const [column, direction] = state.outerOrderBySpecs[i]!
+				return orderKeys !== undefined && mixedNumbers.has(column)
+					? orderKeys(quoteIdent(column)).map((key) => `${key} ${direction.toUpperCase()}`).join(", ")
+					: rendered
+			})
+			sql += `\nORDER BY ${clause.join(", ")}`
 		}
 		if (state.outerLimitValue != null) {
 			sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}`

src/ch/dialect.ts

@@ -151,6 +151,15 @@ export interface Dialect extends Omit<SqlSyntax, "dialect"> {
 	 * unchecked: a custom dialect says which set it renders, if either.
 	 */
 	readonly functions?: string
+	/**
+	 * The ORDER BY keys for a numeric UNION ALL column whose branches may have
+	 * different types. ClickHouse has no common type for `UInt64` and `Int64`
+	 * (or `Int64` and `Float64`): the column is a `Variant`, which ORDER BY
+	 * refuses as a key. The values are left as they are (an INSERT ... SELECT or
+	 * an IN over the union keeps them exact); only the sort key is converted.
+	 * Absent: the dialect orders such a column as it is.
+	 */
+	readonly unionNumberOrderKeys?: (column: string) => ReadonlyArray<string>
 }
 
 /** ClickHouse, with params written into the SQL as literals. The default. */
@@ -173,6 +182,8 @@ export const clickhouseDialect: Dialect = {
 	},
 	transactions: noTransactions,
 	functions: "clickhouse",
+	// CAST, not toFloat64: ClickHouse 26.2's toFloat64 refuses a Variant. Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order.
+	unionNumberOrderKeys: (column) => [`CAST(${column} AS Nullable(Float64))`, `accurateCastOrNull(${column}, 'Int128')`],
 }
 
 // The dialect of the enclosing compile, beside the syntax installed for the

tests/fuzz/README.md

@@ -17,7 +17,8 @@ and are skipped otherwise. Postgres runs in-process on PGlite.
 
 | File | Property | Oracle |
 | --- | --- | --- |
-| `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target |
+| `queries.fuzz.test.ts` | random plans, and `unionAll`s of them (branch and union ORDER BY / LIMIT / OFFSET), return the rows `model.ts` computes | a JS model of SQL, per target |
+| `writes.fuzz.test.ts` | random sequences of inserts (multi-row, defaulted cells, INSERT … SELECT, ON CONFLICT), updates and deletes, with RETURNING, leave a scratch table as the model says; unsupported writes are refused | a JS model of the table's rows |
 | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser |
 | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants |
 | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — |
@@ -36,6 +37,14 @@ A `FuzzDialect` gives:
 - `strings` and `ilike`: what the generators may produce for it.
 - `literals`: its column types, each with a way to build the reference row
   *without* the builder (usually the database parsing JSON).
+- `writes`: which writes its builder compiles (`insertSelect`, `update`, `delete`,
+  `onConflict`, `returning`, `defaultCells` for a multi-row VALUES that leaves a
+  defaulted column out of some rows), and a `scratch` table: its query `Table`,
+  the DDL that creates and drops it, and `exec` to run that DDL on the first
+  target. Set a capability from what the builder does, not what the database
+  could: the suite still generates an unsupported write and requires the builder
+  to refuse it. A database whose writes are asynchronous (ClickHouse mutations)
+  must make them visible to the next read (`mutations_sync=2`).
 - `targets`: the databases it runs on. Each target states its `Semantics`, the
   points where databases legitimately disagree (NULL ordering, integer division,
   what an unmatched LEFT JOIN column holds, LIKE case rules, …). The model reads

tests/fuzz/build.ts

@@ -8,7 +8,7 @@ import { DateTime, Effect, Exit } from "effect"
 import type * as CH from "@maple-dev/effect-orm/clickhouse"
 import type { CoreContext } from "../core-cases"
 import type { FuzzDialect } from "./dialects/types"
-import type { Agg, Cond, Expr, Operand, Plan } from "./plan"
+import type { Agg, Cond, Expr, Operand, Plan, UnionPlan } from "./plan"
 
 const pad = (n: number, width = 2) => String(n).padStart(width, "0")
 /** `YYYY-MM-DD hh:mm:ss.sss` in UTC: the string form a DateTime comparison accepts. */
@@ -18,13 +18,13 @@ export const timeString = (ms: number): string => {
 }
 
 /** Every param operand in the plan, in a fixed walk order, with the name it compiles as. */
-const nameParams = (plan: Plan): { readonly names: Map<Operand, string>; readonly params: Record<string, unknown> } => {
+const nameParams = (plan: Plan, prefix: string): { readonly names: Map<Operand, string>; readonly params: Record<string, unknown> } => {
 	const names = new Map<Operand, string>()
 	const params: Record<string, unknown> = { orgId: plan.orgId }
 	const operand = (op: Operand) => {
 		if (op.k === "expr") return expr(op.e)
 		if (op.k !== "param") return
-		const name = `p${names.size}`
+		const name = `${prefix}p${names.size}`
 		names.set(op, name)
 		params[name] = op.kind === "dateTime" ? DateTime.makeUnsafe(op.v as number) : op.v
 	}
@@ -74,9 +74,10 @@ export interface Built {
 	readonly params: Record<string, unknown>
 }
 
-export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Built => {
+/** `prefix` keeps the params of union branches apart: each branch names its own from `<prefix>p0`. */
+export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan, prefix = ""): Built => {
 	const { lit, param, and, or, not, rawCond, fromQuery } = dialect.module
-	const { names, params } = nameParams(plan)
+	const { names, params } = nameParams(plan, prefix)
 
 	const expr = ($: any, e: Expr): any => {
 		switch (e.k) {
@@ -196,10 +197,9 @@ export type Compiled =
  * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that
  * escapes `compile` — or the builder calls before it — is `Threw`.
  */
-export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => {
+export const classify = (compile: () => Effect.Effect<CH.CompiledQuery<any>, unknown>): Compiled => {
 	try {
-		const { query, params } = buildPlan(dialect, ctx, plan)
-		const exit = Effect.runSyncExit(dialect.module.compile(query, params))
+		const exit = Effect.runSyncExit(compile())
 		if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value }
 		const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail")
 		return failure !== undefined
@@ -209,3 +209,26 @@ export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan):
 		return { _tag: "Threw", error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error) }
 	}
 }
+
+export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled =>
+	classify(() => {
+		const { query, params } = buildPlan(dialect, ctx, plan)
+		return dialect.module.compile(query, params)
+	})
+
+/** A union plan as `unionAll(...)` of its branches, each with its own params. */
+export const buildUnion = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Built => {
+	const built = plan.branches.map((branch, i) => buildPlan(dialect, ctx, branch, `b${i}`))
+	let union: any = (dialect.module.unionAll as (...queries: ReadonlyArray<unknown>) => unknown)(...built.map((b) => b.query))
+	if (plan.orderBy.length > 0) union = union.orderBy(...plan.orderBy.map(([alias, dir]) => [alias, dir]))
+	if (plan.limit !== undefined) union = union.limit(plan.limit)
+	if (plan.offset !== undefined) union = union.offset(plan.offset)
+	if (plan.crossTenant) union = union.crossTenant()
+	return { query: union, params: Object.assign({}, ...built.map((b) => b.params)) }
+}
+
+export const compileUnionPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Compiled =>
+	classify(() => {
+		const { query, params } = buildUnion(dialect, ctx, plan)
+		return dialect.module.compileUnion(query, params)
+	})

tests/fuzz/dialects/clickhouse.ts

@@ -37,8 +37,17 @@ export const clickhouseText = async (
 	}
 }
 
+/**
+ * A ClickHouse server bug, not the builder's: the common-expression rewrite can
+ * leave a one-argument and() (`a AND ((b AND a) OR a)`, `(a OR 0 OR a) AND (a OR 0)`,
+ * and more shapes than a matcher keeps up with), which fails with this error.
+ * The query is run again with the rewrite off, and its rows still checked.
+ */
+const SERVER_AND_BUG = /Number of arguments for function "and" should be at least 2/
+
 const runClickHouse = async (compiled: CH.CompiledQuery<any>, settings: Record<string, string>): Promise<Outcome> => {
-	const { ok, text } = await clickhouseText(compiled.sql, settings)
+	let { ok, text } = await clickhouseText(compiled.sql, settings)
+	if (!ok && SERVER_AND_BUG.test(text)) ({ ok, text } = await clickhouseText(compiled.sql, { ...settings, optimize_extract_common_expressions: "0" }))
 	if (!ok) return { _tag: "DatabaseError", message: text.trim() }
 	const wire = text
 		.split("\n")
@@ -58,12 +67,20 @@ const semantics: Semantics = {
 	likeIgnoresCase: false,
 }
 
+/**
+ * An UPDATE is an `ALTER TABLE … UPDATE` mutation, which runs in the background
+ * unless asked to wait, and a lightweight DELETE may too. Waiting makes a write
+ * visible to the read that follows it, as the writes suite needs.
+ */
+const syncWrites = { mutations_sync: "2", lightweight_deletes_sync: "2" }
+
 const target = (name: string, joinUseNulls: boolean): FuzzTarget => ({
 	name,
 	available: clickhouseEndpoint !== undefined,
 	semantics: joinUseNulls ? { ...semantics, missingJoinColumn: "null" } : semantics,
 	run: (compiled) =>
 		runClickHouse(compiled, {
+			...syncWrites,
 			join_use_nulls: joinUseNulls ? "1" : "0",
 			output_format_json_quote_64bit_integers: joinUseNulls ? "1" : "0",
 		}),
@@ -132,5 +149,26 @@ export const clickhouse: FuzzDialect = {
 	strings,
 	ilike: true,
 	literals,
+	writes: {
+		insertSelect: true,
+		update: true,
+		delete: true,
+		onConflict: false,
+		returning: false,
+		defaultCells: true,
+		scratch: {
+			table: (name) =>
+				CH.table(name, {
+					external: true,
+					columns: { Id: CH.int64, OrgId: CH.string, Name: CH.string, Note: CH.nullable(CH.string), Qty: CH.column(CH.int64, { default: 5 }) },
+				}),
+			create: (name) => [`CREATE TABLE ${name} (Id Int64, OrgId String, Name String, Note Nullable(String), Qty Int64 DEFAULT 5) ENGINE = MergeTree ORDER BY Id`],
+			drop: (name) => `DROP TABLE IF EXISTS ${name}`,
+			exec: async (sql) => {
+				const { ok, text } = await clickhouseText(sql, syncWrites)
+				if (!ok) throw new Error(text.trim())
+			},
+		},
+	},
 	targets: [target("clickhouse", false), target("clickhouse-join-nulls", true)],
 }

tests/fuzz/dialects/postgres.ts

@@ -69,6 +69,26 @@ export const postgres: FuzzDialect = {
 			: `placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${count} parameters`
 	},
 	literals,
+	writes: {
+		insertSelect: true,
+		update: true,
+		delete: true,
+		onConflict: true,
+		returning: true,
+		defaultCells: true,
+		scratch: {
+			table: (name) =>
+				PG.table(name, {
+					external: true,
+					columns: { Id: PG.int8, OrgId: PG.text, Name: PG.text, Note: PG.nullable(PG.text), Qty: PG.column(PG.int8, { default: 5 }) },
+				}),
+			create: (name) => [`CREATE TABLE "${name}" ("Id" int8 PRIMARY KEY, "OrgId" text NOT NULL, "Name" text NOT NULL, "Note" text, "Qty" int8 NOT NULL DEFAULT 5)`],
+			drop: (name) => `DROP TABLE IF EXISTS "${name}"`,
+			exec: async (sql) => {
+				await database().exec(sql)
+			},
+		},
+	},
 	targets: [
 		{
 			name: "postgres",