feat(postgres): typed jsonb paths, @> and jsonb_set from the column schema

.changeset/pg-json-paths.md

@@ -0,0 +1,5 @@
+---
+"@maple-dev/effect-orm": minor
+---
+
+Typed jsonb paths on Postgres, read off the column's schema. `PG.jsonPath($.payload, "profile", "city")` on a `PG.jsonb(Schema.Struct({ profile: Schema.Struct({ city: Schema.String }) }))` column is an `Expr<string | null>` that compiles to `("payload" -> 'profile' -> 'city')` and decodes with the field's own schema; only paths the schema has type-check, and a number key indexes an array. `jsonPathText` reads with `->>`, `jsonContains` filters with `@>` (bound as `$n::jsonb`), and `jsonSet` writes one path with `jsonb_set` inside an UPDATE's `set`, without reading the row first. Values compared with a path, and the value of a `jsonSet`, are encoded by the field's schema and bound as JSON. A path the schema does not have, or one through a field whose stored form differs from its type, is a defect at compile time. The path types (`JsonPath`, `JsonPathValue`, `JsonContained`, …) and `jsonPathSchema` are exported for helpers.

design/dialects.md

@@ -84,5 +84,13 @@ What we deliberately do not copy:
   decision for a later release.
 - The ClickHouse function catalog writes ClickHouse SQL whatever the dialect. Functions could
   refuse to compile under another dialect instead of producing SQL that fails at the server.
+- JSON paths split the same way. `src/ch/json-path.ts` owns what no dialect changes: the path
+  types (`JsonPath`, `JsonPathValue`, `JsonContained`) and `jsonPathSchema`, which reads the
+  decoder at a path off the column's Schema. `src/pg/functions.ts` owns only the SQL (`->`,
+  `->>`, `@>`, `jsonb_set`). A SQLite or MySQL catalog adds its own `jsonPath`, `jsonSet` and
+  friends over the same keys: `json_extract(x, '$.a[0]')` / `x ->> '$.a'` and
+  `json_set(x, '$.a', json(?))` on SQLite; `JSON_EXTRACT`, `JSON_UNQUOTE`,
+  `JSON_CONTAINS(x, CAST(? AS JSON))` and `JSON_SET` on MySQL. The one new piece is a key list
+  written as a `$.a[0]` path string, with each key quoted (`$."a b"`).
 - MySQL or SQLite would need `?` placeholders (`reuse: false` already exists) and their own
   types and functions; nothing in the core should need to change.

design/gap-review.md

@@ -30,7 +30,7 @@ builder; **P1** commonly used; **P2** niche.
 | Postgres column types: `timestamptz` as `Date`, `timestamp`, `date`, `interval`, `varchar(n)`, serial / identity | 226 timestamp columns | S |
 | ~~DISTINCT (and DISTINCT ON)~~ (built) | ~10 | S |
 | ~~`FOR UPDATE` / `FOR SHARE` / `SKIP LOCKED` / `NOWAIT`~~ (built) | 7 | S |
-| jsonb and array operators (`@>`, `->`, `?`, `&&`, `ANY`) | ~12 | M |
+| jsonb and array operators (`@>`, `->`, `?`, `&&`, `ANY`); built: typed `->` / `->>` paths, `@>`, `jsonb_set` (docs/postgres.md#json-paths); open: `?`, `&&`, `ANY` | ~12 | M |
 | ~~`isNull` / `isNotNull` / `between`; variadic `and` / `or` that skip `undefined`~~ (built) | everywhere | S |
 | Constraint error helpers (unique, foreign key, not null); keep ClickHouse's numeric error codes, which `sqlStateOf` drops today | all upserts | S |
 | Tenant-scope enforcement in `Database`, opt in, with an explicit cross-tenant entry point | safety | S |

docs/postgres.md

@@ -182,6 +182,84 @@ either form, and writes its value as array text, which Postgres casts to the col
 work for an enum `generate` does not manage; see [migrations](./migrations.md#postgres) for
 moving one to `enumType`.
 
+## JSON paths
+
+A `jsonb(schema)` column knows the shape of its documents, so a path into one is typed and
+decoded by the same schema. `jsonPath(column, ...keys)` reads the value at a path with `->`;
+a string key steps into an object, a number into an array. Only paths the schema has
+type-check, and the result decodes with the field's own schema (a `NumberFromString` field
+reads as a number). A key that is absent reads as `null`.
+
+```ts title="postgres-json-paths.ts"
+import { PGlite } from "@electric-sql/pglite"
+import { Effect, Schema } from "effect"
+import * as PG from "@maple-dev/effect-orm/postgres"
+
+const Settings = Schema.Struct({
+	notifications: Schema.Struct({ sms: Schema.Boolean, email: Schema.Boolean }),
+	tags: Schema.Array(Schema.String),
+})
+
+const Users = PG.table("users", {
+	columns: { id: PG.int4, settings: PG.jsonb(Settings) },
+	primaryKey: ["id"],
+})
+
+// Read and filter: `sms` is an Expr<boolean | null>.
+export const read = PG.compileUnsafe(
+	PG.from(Users)
+		.select(($) => ({ id: $.id, sms: PG.jsonPath($.settings, "notifications", "sms") }))
+		.where(($) => [PG.jsonPath($.settings, "notifications", "email").eq(true), PG.jsonContains($.settings, { tags: ["beta"] })]),
+	{},
+)
+// read.sql:        SELECT "users"."id" AS "id", ("users"."settings" -> 'notifications' -> 'sms') AS "sms" FROM "users"
+//                  WHERE ("users"."settings" -> 'notifications' -> 'email') = $1 AND ("users"."settings" @> $2::jsonb)
+// read.parameters: ["true", '{"tags":["beta"]}']
+
+// Write one path, without reading the row first.
+export const write = PG.compileUnsafe(
+	PG.update(Users)
+		.set(($) => ({ settings: PG.jsonSet($.settings, ["notifications", "sms"], false) }))
+		.where(($) => [$.id.eq(1)]),
+	{},
+)
+// write.sql: UPDATE "users" SET "settings" = jsonb_set("settings", ARRAY['notifications', 'sms'], $1::jsonb) WHERE "id" = $2
+
+const db = new PGlite()
+await db.exec(`
+	CREATE TABLE users (id int4 PRIMARY KEY, settings jsonb NOT NULL);
+	INSERT INTO users VALUES (1, '{"notifications": {"sms": true, "email": true}, "tags": ["beta"]}');
+`)
+await db.query(write.sql, [...write.parameters])
+const result = await db.query<Record<string, unknown>>(read.sql, [...read.parameters])
+export const rows = await Effect.runPromise(read.decodeRows(result.rows))
+// [{ id: 1, sms: false }]
+await db.close()
+```
+
+| Function | SQL | Result |
+| --- | --- | --- |
+| `jsonPath(x, ...keys)` | `(x -> 'k1' -> 'k2')` | the field's type, or `null` |
+| `jsonPathText(x, ...keys)` | `(x -> 'k1' ->> 'k2')` | `string \| null`: a JSON string without its quotes |
+| `jsonContains(x, part)` | `(x @> $n::jsonb)` | a condition |
+| `jsonSet(x, [keys], value)` | `jsonb_set(x, ARRAY['k1', 'k2'], $n::jsonb)` | the document's type |
+
+- A value compared with a `jsonPath`, and the value of a `jsonSet`, is encoded by the field's
+  schema and bound as JSON text: `.eq("Kyiv")` matches the JSON string `"Kyiv"`. For text
+  comparisons and `like`, use `jsonPathText`.
+- `jsonContains` takes part of the document, at any depth: objects match on the keys given,
+  arrays on the elements given, in any order. It accepts a `jsonPath` too. Its value is written
+  as given, so a field whose schema transforms (a date stored as text) is written in its stored
+  form. A GIN index on the column serves `@>`.
+- `jsonSet` creates the last key if it is absent. A missing parent leaves the document as it is,
+  and a `NULL` document stays `NULL`. Nest calls to set more than one path.
+- Keys are written into the SQL as literals, not bound, so an expression index on
+  `(settings -> 'notifications')` matches.
+- The schema walk follows structs, records, arrays and tuples, through unions, `NullOr`,
+  optional keys and `Schema.suspend`. A path through a field whose stored form is not its
+  type (`Schema.encodeKeys`, a class) is refused when the query compiles. A `jsonb()` column
+  with no schema takes any keys and reads `unknown`.
+
 ## Functions
 
 | Function | SQL | Notes |
@@ -197,6 +275,7 @@ moving one to `enumType`.
 | `lower`, `upper`, `length` | same | |
 | `coalesce(x, fallback)` | `coalesce(x, fallback)` | No longer nullable |
 | `jsonText(x, key)` | `(x ->> key)` | `null` when absent |
+| `jsonPath`, `jsonPathText`, `jsonContains`, `jsonSet` | `->`, `->>`, `@>`, `jsonb_set` | Typed by the column schema; see [JSON paths](#json-paths) |
 | `greatest(a, ...)`, `least(a, ...)` | same | NULL arguments are skipped |
 | `caseWhen([[c, v], ...], otherwise)` | `CASE WHEN c THEN v ... ELSE otherwise END` | |
 | `asBoolean(c)` | `(c)` | A condition as a value, to select or `set` |

docs/reference.md

@@ -489,9 +489,12 @@ one-dimensional array literal (`{a,"b c",NULL}`), the form enum arrays arrive in
 `max(x)`, `percentileCont(f, x)`, `arrayAgg(x)`, `dateTrunc(unit, ts)` (`DateTruncUnit` is the
 unit union), `dateBin(seconds, ts)`, `now()`, `lower(x)`, `upper(x)`, `length(x)`,
 `coalesce(x, fallback)`, `nullIf(x, value)`, `jsonText(x, key)`, `greatest(a, ...)`, `least(a, ...)`,
+`jsonPath(x, ...keys)`, `jsonPathText(x, ...keys)`, `jsonContains(x, part)`, `jsonSet(x, keys, value)`
+(typed jsonb paths; the types `JsonKey`, `JsonKeyOf`, `JsonStep`, `JsonPath`, `JsonPathValue` and
+`JsonContained` spell them, and `jsonPathSchema` reads a path's schema off a document schema, as a `JsonPathSchema`),
 `caseWhen([[c, v], ...], otherwise)`, `asBoolean(c)` (a condition as a value) and
 `typedValue(type, value)` (a value bound as a column type writes it). See
-[Postgres functions](./postgres.md#functions) for the SQL each writes.
+[Postgres functions](./postgres.md#functions) and [JSON paths](./postgres.md#json-paths) for the SQL each writes.
 
 ### Dialect
 

scripts/check-doc-examples.mjs

@@ -117,6 +117,12 @@ assert.deepEqual(enums.shared.parameters, ["public", "private"])
 assert.deepEqual(await Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "public" }])), [{ id: "c", type: "public" }])
 await assert.rejects(() => Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "secret" }])))
 assert.match(sql(enums.upsert), /ON CONFLICT \\("slug"\\) DO UPDATE SET/)
+const json = await import("./postgres-json-paths")
+assert.ok(sql(json.read).includes("(\\"users\\".\\"settings\\" -> 'notifications' -> 'sms') AS \\"sms\\""))
+assert.ok(sql(json.read).includes("(\\"users\\".\\"settings\\" @> $2::jsonb)"))
+assert.deepEqual(json.read.parameters, ["true", '{"tags":["beta"]}'])
+assert.equal(sql(json.write), "UPDATE \\"users\\" SET \\"settings\\" = jsonb_set(\\"settings\\", ARRAY['notifications', 'sms'], $1::jsonb) WHERE \\"id\\" = $2")
+assert.deepEqual(json.rows, [{ id: 1, sms: false }])
 const database = await import("./database-transaction")
 assert.deepEqual(database.balances, { from: 70, to: 30, refused: "InsufficientFunds" })
 console.log("Markdown example behavior checks passed")

src/ch/json-path.ts

@@ -0,0 +1,127 @@
+// Typed JSON paths, the part no dialect changes.
+//
+// A path is a list of keys: a string steps into an object, a number into an
+// array. Its type comes from the column's TypeScript type (`JsonPath`,
+// `JsonPathValue`) and its decoder from the column's Schema (`jsonPathSchema`),
+// so `jsonPath($.payload, "profile", "city")` is an `Expr<string | null>` that
+// validates what it reads. Only the SQL differs per dialect: Postgres writes
+// `x -> 'profile' -> 'city'` (src/pg/functions.ts); a SQLite or MySQL catalog
+// would write `json_extract(x, '$.profile.city')` from the same keys and reuse
+// everything here.
+
+import { Schema, SchemaAST } from "effect"
+
+/** One step into a JSON document: an object key or an array index. */
+export type JsonKey = string | number
+
+/** Counts down the depth `JsonPath` spells out, so a recursive type ends. */
+type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+/** The keys one step into a value of type `A`: its field names, or `number` for an array. */
+export type JsonKeyOf<A> = unknown extends A
+	? JsonKey
+	: A extends ReadonlyArray<any>
+		? number
+		: A extends object
+			? Extract<keyof A, string>
+			: never
+
+/** The value one key into `A`; `never` when `A` has no such key. */
+export type JsonStep<A, K> = unknown extends A
+	? unknown
+	: A extends ReadonlyArray<infer E>
+		? K extends number
+			? E
+			: never
+		: A extends object
+			? K extends keyof A
+				? A[K]
+				: never
+			: never
+
+/**
+ * Every path into a value of type `A`, as a tuple of keys, up to eight deep.
+ * A document typed `unknown` (`jsonb()` with no schema) takes any keys.
+ */
+export type JsonPath<A, D extends number = 8> = [D] extends [never]
+	? ReadonlyArray<JsonKey>
+	: unknown extends A
+		? readonly [JsonKey, ...Array<JsonKey>]
+		: { [K in JsonKeyOf<A>]: readonly [K] | readonly [K, ...JsonPath<JsonStep<A, K>, Prev[D]>] }[JsonKeyOf<A>]
+
+/** The value at `path` inside `A`, before SQL adds `null` for a missing key. */
+export type JsonPathValue<A, P> = P extends readonly [infer K, ...infer Rest] ? JsonPathValue<JsonStep<A, K>, Rest> : A
+
+/**
+ * What `jsonContains` takes: part of a document, at any depth. Containment is
+ * read on the stored JSON, so a field whose schema transforms (a `Date` stored
+ * as a string) is written in its stored form.
+ */
+export type JsonContained<A> = unknown extends A
+	? unknown
+	: A extends ReadonlyArray<infer E>
+		? ReadonlyArray<JsonContained<E>>
+		: A extends object
+			? { readonly [K in keyof A]?: JsonContained<A[K]> }
+			: A
+
+/** The schema at `path`, or why there is none. */
+export type JsonPathSchema =
+	| { readonly _tag: "Found"; readonly schema: Schema.Codec<unknown, unknown> }
+	| { readonly _tag: "Missing"; readonly reason: string }
+
+const missing = (reason: string): JsonPathSchema => ({ _tag: "Missing", reason })
+
+/** A union's members that hold a value: `null` and `undefined` are where a key reads as SQL NULL. */
+const valueMembers = (types: ReadonlyArray<SchemaAST.AST>): ReadonlyArray<SchemaAST.AST> =>
+	types.filter((member) => member._tag !== "Null" && member._tag !== "Undefined")
+
+/**
+ * The node one key into `ast`. A node with its own encoding is refused: the
+ * document stores its encoded form, whose keys need not be the decoded ones
+ * (`Schema.encodeKeys`, a class), so the path would read the wrong field.
+ */
+const step = (ast: SchemaAST.AST, key: JsonKey): SchemaAST.AST | string => {
+	if (ast._tag === "Unknown" || ast._tag === "Any") return ast
+	if (ast._tag === "Suspend") return step(ast.thunk(), key)
+	if (ast.encoding !== undefined) return "its stored form is a transformation of its type"
+	switch (ast._tag) {
+		case "Union": {
+			const found = valueMembers(ast.types).map((member) => step(member, key))
+			const asts = found.filter((result): result is SchemaAST.AST => typeof result !== "string")
+			if (asts.length === 0) return found.find((result) => typeof result === "string") ?? "no member holds a value"
+			return asts.length === 1 ? asts[0]! : Schema.Union(asts.map((member) => Schema.make(member))).ast
+		}
+		case "Objects": {
+			if (typeof key !== "string") return `an object has no index ${key}`
+			const field = ast.propertySignatures.find((signature) => signature.name === key)
+			if (field !== undefined) return field.type
+			const index = ast.indexSignatures[0]
+			return index !== undefined ? index.type : `no field "${key}"`
+		}
+		case "Arrays": {
+			if (typeof key !== "number") return `an array has no field "${key}"`
+			const element = (key >= 0 ? ast.elements[key] : undefined) ?? ast.rest[0]
+			return element ?? `no element ${key}`
+		}
+		default:
+			return `a ${ast._tag} has no fields`
+	}
+}
+
+/**
+ * The schema of the value at `path` inside a document `schema` decodes, read
+ * off the schema itself: struct fields, records, arrays and tuples, through
+ * unions, `NullOr`, optional keys and `suspend`. A document typed `Unknown`
+ * reads `Unknown` at every path. The caller adds `null`, which SQL returns for
+ * a missing key.
+ */
+export const jsonPathSchema = (schema: Schema.Top, path: ReadonlyArray<JsonKey>): JsonPathSchema => {
+	let ast: SchemaAST.AST = schema.ast
+	for (const [depth, key] of path.entries()) {
+		const next = step(ast, key)
+		if (typeof next === "string") return missing(`at ${JSON.stringify(path.slice(0, depth + 1))}: ${next}`)
+		ast = next
+	}
+	return { _tag: "Found", schema: Schema.make(ast) as unknown as Schema.Codec<unknown, unknown> }
+}