feat(postgres): client-side defaults as Effects with defaultFn and onUpdate

.changeset/client-defaults.md

@@ -0,0 +1,5 @@
+---
+"@maple-dev/effect-orm": minor
+---
+
+`PG.column` takes client-side defaults as Effects: `defaultFn` runs for each row an insert leaves the column out of, and `onUpdate` for each update that does not set it (and for an insert, when the column has no other default). An Effect can read the `Clock`, `Random` or a service, so `TestClock` and a test layer give exact values in tests. Such a column is optional in the insert row type, and the Effect's requirements become the write's: `Db.run(PG.insertInto(Notes).values(...))` requires `IdGen` when `id` is `{ defaultFn: IdGen.next }`. `Database.run` runs them; `fillDefaults(write)` runs them for a write you compile yourself, and `compile` refuses a write that still leaves one out. `Table` and `PgSchemaTable` gain two type parameters for the requirements; they default to `never`.

design/gap-review.md

@@ -45,7 +45,8 @@ builder; **P1** commonly used; **P2** niche.
 - ClickHouse: FINAL, PREWHERE, LIMIT BY, SETTINGS on SELECT, the ARRAY JOIN clause.
 - Writes: UPDATE ... FROM with a typed VALUES source; ClickHouse `ALTER TABLE ... UPDATE/DELETE`
   and lightweight DELETE; JSONEachRow bulk insert (`encodeInsertRows`); a set-all-from-`excluded`
-  helper; client-side default hooks like `$defaultFn` / `$onUpdate`.
+  helper; ~~client-side default hooks like `$defaultFn` / `$onUpdate`~~ (built: `defaultFn` /
+  `onUpdate` on `PG.column`, as Effects).
 - Runtime: `runFirst` / `runSingle` / `stream`; `observe` with duration, rows and error; replica
   routing for `route()`; a `SET LOCAL` helper for RLS; a mock `Database` for users' tests.
 - Types: select, insert and update row types and Schemas per table.

docs/database.md

@@ -83,7 +83,7 @@ Calling `withdraw(1, 30)` outside `transfer` does not compile: `requireTransacti
 | `fromSqlClient(sql, options)` | A `DatabaseApi` over a client. `options.dialect` is required; `options.requireTenantScope` refuses unscoped queries ([Tenant scoping](./tenant-scoping.md#enforcing-it)) |
 | `layerSqlClient(options)` | A `Database` layer over the `SqlClient` in context |
 | `Database` | The service |
-| `run(query, params?)` | Compile a query for the database's dialect, run it, decode its rows |
+| `run(query, params?)` | Compile a query for the database's dialect, run it, decode its rows. A write first runs its table's [client-side defaults](./inserts.md#client-side-defaults) and requires what they need |
 | `sql\`...\`` | A statement with every `${value}` bound; `sql.identifier(name)` for a table or column name |
 | `query(statement, schema?)` | Run a statement and return its rows, decoded when a schema is given |
 | `execute(statement)` | Run a statement and discard its rows |

docs/inserts.md

@@ -51,6 +51,8 @@ The row type is derived from the column options that also write the [DDL](./migr
 it cannot drift from what the database fills in:
 
 - `default` or `defaultExpr` (both dialects) or `identity` (Postgres) makes a column optional.
+- `defaultFn` or `onUpdate` (Postgres) makes a column optional too. The client fills it, not the
+  database: see [client-side defaults](#client-side-defaults).
 - `materialized` or `alias` (ClickHouse) makes a column not writable: it is not in the row type,
   and a row that names it anyway fails to compile. It stays readable.
 
@@ -67,6 +69,50 @@ ClickHouse fills every column it is not given with a default, even without a `DE
 `0` for a number, `''` for a string. The row type still requires those columns unless they
 declare a default, so a forgotten value is a type error rather than a silent zero.
 
+### Client-side defaults
+
+A `PG.column` can take a default that the client computes: `defaultFn` and `onUpdate` are
+Effects, so they can read the `Clock`, `Random` or a service. They write no DDL.
+
+```ts
+import { Context, DateTime, Effect } from "effect"
+import * as Db from "@maple-dev/effect-orm/database"
+import * as PG from "@maple-dev/effect-orm/postgres"
+
+class IdGen extends Context.Service<IdGen, { readonly next: Effect.Effect<string> }>()("IdGen") {}
+
+const Notes = PG.table("notes", {
+	columns: {
+		id: PG.column(PG.text, { defaultFn: Effect.flatMap(Effect.service(IdGen), (gen) => gen.next) }),
+		body: PG.text,
+		createdAt: PG.column(PG.timestamptz, { defaultFn: DateTime.now }),
+		updatedAt: PG.column(PG.timestamptz, { onUpdate: DateTime.now }),
+	},
+	primaryKey: ["id"],
+})
+
+// Effect<ReadonlyArray<never>, ..., Database | IdGen>
+const add = Db.run(PG.insertInto(Notes).values({ body: "hello" }))
+```
+
+- `Database.run` runs `defaultFn` once for each row that leaves its column out (or gives
+  `undefined`). A value the row gives, `null` included, is kept, and the Effect does not run.
+- `onUpdate` runs once for each update that does not set its column (see
+  [Updating](./updates-and-deletes.md#set)). An insert that leaves the column out runs it too,
+  when the column has no `defaultFn`, `default`, `defaultExpr` or `identity`.
+- What the Effects need is in the type of the write: the insert above requires `IdGen`, and
+  `Db.run` passes the requirement on. `DateTime.now` needs only the `Clock`, which every Effect
+  has, so it adds nothing. In a test, `TestClock` and a test `IdGen` layer give exact values.
+- The Effects cannot fail: their error type is `never`. A default that can fail should handle
+  the failure itself.
+- `compile` does not run Effects. To compile such a write yourself, pass it through
+  `fillDefaults` first: `Effect.flatMap(PG.fillDefaults(insert), (filled) => PG.compile(filled))`.
+  `compile` refuses (`QueryBuilderDefect`) a write that still leaves out a client default.
+- An `INSERT ... SELECT` and `onConflictDoUpdate`'s SET get no client defaults; select or set
+  the column yourself.
+
+_(Backed by `src/database/client-defaults.test.ts` and `src/database/client-defaults.test-d.ts`.)_
+
 ## What it compiles to
 
 Columns are written in table order, whatever order the keys are in, so two rows with their keys
@@ -240,6 +286,7 @@ A batch that mixes tenants on purpose, such as an ingest flush, declares it with
 | A param with no value                             | `QueryBuilderError` `UnresolvedParam`    |
 | Over the dialect's bound-value limit              | `QueryBuilderError` `InvalidArguments`   |
 | Compiling without `values`                        | `QueryBuilderDefect`                     |
+| Compiling a row that leaves out a client default  | `QueryBuilderDefect`                     |
 | `returning` for a dialect without RETURNING       | `QueryBuilderDefect`                     |
 | `onConflictDoUpdate` setting no or unknown columns | `QueryBuilderError` `InvalidArguments`  |
 | `onConflict*` without ON CONFLICT, a bad target   | `QueryBuilderDefect`                     |

docs/reference.md

@@ -47,6 +47,7 @@ Everything in this section is exported, identically, from both `/clickhouse` and
 | `update`     | `(table) => CHUpdateStart`, then `CHUpdate`: `.set(record \| fn)`, `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)`. See [Updating and deleting](./updates-and-deletes.md) |
 | `deleteFrom` | `(table) => CHDelete`: `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)` |
 | `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres). See [Inserting rows](./inserts.md) |
+| `fillDefaults` | `(write) => Effect<write, never, RequirementsOf<write>>`: runs the `defaultFn` / `onUpdate` Effects of an insert's or update's table into its rows or SET. `Database.run` calls it; call it before `compile` when you compile a write yourself. See [client-side defaults](./inserts.md#client-side-defaults) |
 
 Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or
 [`PG.table`](#tables-and-ddl-1).
@@ -181,6 +182,10 @@ decoded type of a column), `InferEncoded` (its wire type), `ColumnDefs`, `Nullab
 **Inference** — `InferOutput`, `InferQueryOutput`, `InferUnionOutput`, `SelectRowOf`,
 `InsertRow`, `InsertRowOf`, `UpdateSet`, `UpdateSetOf`.
 
+**Client-side defaults** — `RequirementsOf<W>` (what a write's client defaults need; `never` for
+anything else), `ClientDefaults` (the `insert` and `update` Effects per column, on
+`Table.clientDefaults`).
+
 **What `compile` checks** — `NeedsSelect<Output>` (the query has a SELECT list) and
 `ParamsSatisfied<Params, Given>` (the params object fills every required param). A helper generic
 over a query's output takes `CHQuery<…, Output> & NeedsSelect<Output>` so it can pass the query on
@@ -453,7 +458,11 @@ const Users = PG.table("users", {
 `/clickhouse`: a view, a catalog table, one another tool migrates. No DDL.
 
 **`ColumnOptions`** — one of `default`, `defaultExpr`, `identity` (`"always"` or
-`"by default"`); any of them lets an insert leave the column out (`DefaultedColumnsOf`). Also
+`"by default"`); any of them lets an insert leave the column out (`DefaultedColumnsOf`).
+`defaultFn` and `onUpdate` are client-side defaults: an `Effect` of a value of the column's type
+that cannot fail, run by `Database.run` for an insert that leaves the column out and an update
+that does not set it. Either one lets an insert leave the column out, and its requirements
+become the write's ([client-side defaults](./inserts.md#client-side-defaults)). Also
 `name` (the column's name in SQL) and `unique` (`true`, a constraint name, or
 `UniqueColumnOptions`: `{ name?, nullsNotDistinct? }`), which names a one-column constraint
 `<table>_<column>_unique` by default.

docs/updates-and-deletes.md

@@ -35,6 +35,13 @@ names the record type.
 
 `update(table)` offers only `set` until it has one (its type is `CHUpdateStart`).
 
+A Postgres column with `onUpdate` (an Effect, see
+[client-side defaults](./inserts.md#client-side-defaults)) is set by every update that does not
+set it: `Database.run` runs the Effect once and adds the value to the SET. With a SET callback,
+the Effect runs before the callback, and a value the callback gives wins. The update requires
+what the Effect needs. `compile` refuses an update that does not set the column, unless it went
+through `fillDefaults` first.
+
 ## WHERE, and writing every row
 
 `where` works as in a query: a list of conditions, AND-joined, with an `undefined` one skipped,

src/ch/client-defaults.ts

@@ -0,0 +1,75 @@
+// Client-side defaults
+//
+// A Postgres column's `defaultFn` and `onUpdate` are Effects. `fillDefaults`
+// runs them into an insert's rows or an update's SET, so the write that
+// `compile` sees already has every value. `Database.run` calls it first; a
+// write compiled without it, that still lacks a client default, is a defect.
+// Running here, and not in `compile`, keeps `compile` free of requirements.
+
+import { Effect } from "effect"
+import { isInsert, type AnyInsert, type CHInsert } from "./insert"
+import type { ClientDefaults } from "./table"
+import { isUpdate, type CHUpdate } from "./update"
+
+/** What a write's client defaults need to run: `never` for any other runnable. */
+export type RequirementsOf<W> = W extends { readonly _phantom?: { readonly requirements: infer R } } ? R : never
+
+type Row = Readonly<Record<string, unknown>>
+
+/** The values a row leaves out, each from its Effect, in table order. */
+const fillRow = (row: Row, defaults: ClientDefaults["insert"]): Effect.Effect<Row, never, any> => {
+	const missing = Object.entries(defaults).filter(([column]) => row[column] === undefined)
+	if (missing.length === 0) return Effect.succeed(row)
+	return Effect.map(
+		Effect.forEach(missing, ([column, effect]) => Effect.map(effect, (value) => [column, value] as const)),
+		(filled) => ({ ...row, ...Object.fromEntries(filled) }),
+	)
+}
+
+/**
+ * Run the client defaults of a write's table: each `defaultFn` once for every
+ * row of an insert that leaves its column out (an `onUpdate` stands in for a
+ * column without a default), and each `onUpdate` once for an update that does
+ * not set its column. A given value, `null` included, is kept. Any other
+ * runnable, and an `INSERT ... SELECT`, comes back unchanged.
+ *
+ * `Database.run` does this itself. Call it before `compile` when you compile a
+ * write yourself.
+ */
+export const fillDefaults = <W>(write: W): Effect.Effect<W, never, RequirementsOf<W>> => {
+	if (isInsert(write)) return fillInsert(write) as Effect.Effect<any, never, any>
+	if (isUpdate(write)) return fillUpdate(write) as Effect.Effect<any, never, any>
+	return Effect.succeed(write)
+}
+
+const fillInsert = (insert: AnyInsert): Effect.Effect<AnyInsert, never, any> => {
+	const defaults = insert._state.table.clientDefaults?.insert
+	const rows = insert._state.rows
+	if (defaults === undefined || rows === undefined) return Effect.succeed(insert)
+	return Effect.map(Effect.forEach(rows, (row) => fillRow(row, defaults)), (filled) =>
+		// Unchanged when no row left a client default out.
+		filled.every((row, index) => row === rows[index])
+			? insert
+			: ((insert as CHInsert<any, any, any, any, any, any>).values(filled as never) as AnyInsert),
+	)
+}
+
+const fillUpdate = (update: CHUpdate<any, any, any, any, any, any>): Effect.Effect<CHUpdate<any, any, any, any, any, any>, never, any> => {
+	const defaults = update._state.table.clientDefaults?.update
+	const set = update._state.set
+	if (defaults === undefined || set === undefined || Object.keys(defaults).length === 0) return Effect.succeed(update)
+	// A SET record says which columns it writes; a callback says so only when
+	// compiled, so every `onUpdate` runs and the callback's own values win.
+	const given = typeof set === "function" ? {} : (set as Row)
+	return Effect.map(fillRow(given, defaults), (filled) => {
+		const added = Object.fromEntries(Object.entries(filled).filter(([column]) => given[column] === undefined))
+		const next =
+			typeof set === "function"
+				? ($: any) => {
+						const record = set($) as Row
+						return { ...record, ...Object.fromEntries(Object.entries(added).filter(([column]) => record[column] === undefined)) }
+					}
+				: { ...set, ...added }
+		return update.set(next as never) as CHUpdate<any, any, any, any, any, any>
+	})
+}

src/ch/compile.ts

@@ -619,7 +619,7 @@ export function compileCH(
 /** A write statement: what `compile` takes besides a query. */
 export type CHWrite<Output, Params = never> =
 	| AnyInsert<Output, Params>
-	| CHUpdate<any, any, Output, "ready", Params>
+	| CHUpdate<any, any, Output, "ready", Params, any>
 	| CHDelete<any, Output, "ready", Params>
 
 /** What compiling a write takes: only the dialect. */
@@ -1913,6 +1913,10 @@ function compileInsert(insert: CHInsert<any, any, any, any>, params: Record<stri
 	if (rows === undefined && selectQuery === undefined) {
 		return fail(new QueryBuilderDefect({ message: `${where}: values() or select() is required` }),failedQuery<any, undefined>())
 	}
+	const unfilled = rows?.flatMap((row, index) => missingClientDefaults(table.clientDefaults?.insert, row).map((column) => `${column} in row ${index}`))
+	if (unfilled !== undefined && unfilled.length > 0) {
+		return fail(new QueryBuilderDefect({ message: clientDefaultsMessage(where, "insert", unfilled) }), failedQuery<any, undefined>())
+	}
 	const dialect = currentDialect()
 	const { values, cell } = valueCells(table, params, where)
 
@@ -2001,6 +2005,15 @@ function compileInsert(insert: CHInsert<any, any, any, any>, params: Record<stri
 	return writeCompiledQuery(rendered, "insert", own.scope, own.bound, returning, insert._state.crossTenant, table.name)
 }
 
+/** The columns with a client default that a row or SET record leaves out. */
+const missingClientDefaults = (
+	defaults: Readonly<Record<string, unknown>> | undefined,
+	record: Readonly<Record<string, unknown>>,
+): ReadonlyArray<string> => (defaults === undefined ? [] : Object.keys(defaults).filter((column) => record[column] === undefined))
+
+const clientDefaultsMessage = (where: string, kind: "insert" | "update", columns: ReadonlyArray<string>): string =>
+	`${where}: ${columns.join(", ")} ${columns.length === 1 ? "has" : "have"} a client default (defaultFn / onUpdate) that has not run; run the ${kind} with Database.run, or pass it through fillDefaults before compile`
+
 /** The VALUES rows' columns, in table order, after checking every key. */
 const valuesColumns = (
 	table: CHInsert<any, any, any, any>["_state"]["table"],
@@ -2078,6 +2091,10 @@ function compileUpdateOrDelete(
 				const set = (write as CHUpdate<any, any, any>)._state.set
 				if (set === undefined) return fail(new QueryBuilderDefect({ message: `${where}: set() is required` }), [assignments, ""] as const)
 				const record = typeof set === "function" ? set($) : set
+				const unfilled = missingClientDefaults(table.clientDefaults?.update, record as Record<string, unknown>)
+				if (unfilled.length > 0) {
+					return fail(new QueryBuilderDefect({ message: clientDefaultsMessage(where, "update", unfilled) }), [assignments, ""] as const)
+				}
 				assignments = setAssignments(table, record as Record<string, unknown>, cell, (column, value, sql) => {
 					if (column === tenant) setWrites.push({ value, sql })
 				}, where, "update")