From 4f82f611af7fc6fb28dcc1e2c827c15532960724 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:36:18 +0200 Subject: [PATCH 1/4] feat(postgres): typed jsonb paths, @> containment and jsonb_set from the column schema --- src/ch/json-path.ts | 127 +++++++++++++++++++++++++++++ src/pg/functions.ts | 132 +++++++++++++++++++++++++++++- src/pg/json.test-d.ts | 57 +++++++++++++ src/pg/json.test.ts | 183 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 497 insertions(+), 2 deletions(-) create mode 100644 src/ch/json-path.ts create mode 100644 src/pg/json.test-d.ts create mode 100644 src/pg/json.test.ts diff --git a/src/ch/json-path.ts b/src/ch/json-path.ts new file mode 100644 index 0000000..2ee7ee7 --- /dev/null +++ b/src/ch/json-path.ts @@ -0,0 +1,127 @@ +// Typed JSON paths, the part no dialect changes. +// +// A path is a list of keys: a string steps into an object, a number into an +// array. Its type comes from the column's TypeScript type (`JsonPath`, +// `JsonPathValue`) and its decoder from the column's Schema (`jsonPathSchema`), +// so `jsonPath($.payload, "profile", "city")` is an `Expr` that +// validates what it reads. Only the SQL differs per dialect: Postgres writes +// `x -> 'profile' -> 'city'` (src/pg/functions.ts); a SQLite or MySQL catalog +// would write `json_extract(x, '$.profile.city')` from the same keys and reuse +// everything here. + +import { Schema, SchemaAST } from "effect" + +/** One step into a JSON document: an object key or an array index. */ +export type JsonKey = string | number + +/** Counts down the depth `JsonPath` spells out, so a recursive type ends. */ +type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8] + +/** The keys one step into a value of type `A`: its field names, or `number` for an array. */ +export type JsonKeyOf = unknown extends A + ? JsonKey + : A extends ReadonlyArray + ? number + : A extends object + ? Extract + : never + +/** The value one key into `A`; `never` when `A` has no such key. */ +export type JsonStep = unknown extends A + ? unknown + : A extends ReadonlyArray + ? K extends number + ? E + : never + : A extends object + ? K extends keyof A + ? A[K] + : never + : never + +/** + * Every path into a value of type `A`, as a tuple of keys, up to eight deep. + * A document typed `unknown` (`jsonb()` with no schema) takes any keys. + */ +export type JsonPath = [D] extends [never] + ? ReadonlyArray + : unknown extends A + ? readonly [JsonKey, ...Array] + : { [K in JsonKeyOf]: readonly [K] | readonly [K, ...JsonPath, Prev[D]>] }[JsonKeyOf] + +/** The value at `path` inside `A`, before SQL adds `null` for a missing key. */ +export type JsonPathValue = P extends readonly [infer K, ...infer Rest] ? JsonPathValue, Rest> : A + +/** + * What `jsonContains` takes: part of a document, at any depth. Containment is + * read on the stored JSON, so a field whose schema transforms (a `Date` stored + * as a string) is written in its stored form. + */ +export type JsonContained = unknown extends A + ? unknown + : A extends ReadonlyArray + ? ReadonlyArray> + : A extends object + ? { readonly [K in keyof A]?: JsonContained } + : A + +/** The schema at `path`, or why there is none. */ +export type JsonPathSchema = + | { readonly _tag: "Found"; readonly schema: Schema.Codec } + | { readonly _tag: "Missing"; readonly reason: string } + +const missing = (reason: string): JsonPathSchema => ({ _tag: "Missing", reason }) + +/** A union's members that hold a value: `null` and `undefined` are where a key reads as SQL NULL. */ +const valueMembers = (types: ReadonlyArray): ReadonlyArray => + types.filter((member) => member._tag !== "Null" && member._tag !== "Undefined") + +/** + * The node one key into `ast`. A node with its own encoding is refused: the + * document stores its encoded form, whose keys need not be the decoded ones + * (`Schema.encodeKeys`, a class), so the path would read the wrong field. + */ +const step = (ast: SchemaAST.AST, key: JsonKey): SchemaAST.AST | string => { + if (ast._tag === "Unknown" || ast._tag === "Any") return ast + if (ast._tag === "Suspend") return step(ast.thunk(), key) + if (ast.encoding !== undefined) return "its stored form is a transformation of its type" + switch (ast._tag) { + case "Union": { + const found = valueMembers(ast.types).map((member) => step(member, key)) + const asts = found.filter((result): result is SchemaAST.AST => typeof result !== "string") + if (asts.length === 0) return found.find((result) => typeof result === "string") ?? "no member holds a value" + return asts.length === 1 ? asts[0]! : Schema.Union(asts.map((member) => Schema.make(member))).ast + } + case "Objects": { + if (typeof key !== "string") return `an object has no index ${key}` + const field = ast.propertySignatures.find((signature) => signature.name === key) + if (field !== undefined) return field.type + const index = ast.indexSignatures[0] + return index !== undefined ? index.type : `no field "${key}"` + } + case "Arrays": { + if (typeof key !== "number") return `an array has no field "${key}"` + const element = (key >= 0 ? ast.elements[key] : undefined) ?? ast.rest[0] + return element ?? `no element ${key}` + } + default: + return `a ${ast._tag} has no fields` + } +} + +/** + * The schema of the value at `path` inside a document `schema` decodes, read + * off the schema itself: struct fields, records, arrays and tuples, through + * unions, `NullOr`, optional keys and `suspend`. A document typed `Unknown` + * reads `Unknown` at every path. The caller adds `null`, which SQL returns for + * a missing key. + */ +export const jsonPathSchema = (schema: Schema.Top, path: ReadonlyArray): JsonPathSchema => { + let ast: SchemaAST.AST = schema.ast + for (const [depth, key] of path.entries()) { + const next = step(ast, key) + if (typeof next === "string") return missing(`at ${JSON.stringify(path.slice(0, depth + 1))}: ${next}`) + ast = next + } + return { _tag: "Found", schema: Schema.make(ast) as unknown as Schema.Codec } +} diff --git a/src/pg/functions.ts b/src/pg/functions.ts index 827d894..a8cc709 100644 --- a/src/pg/functions.ts +++ b/src/pg/functions.ts @@ -8,11 +8,12 @@ import { fail } from "../ch/failure" import { Schema, type DateTime } from "effect" -import { QueryBuilderDefect } from "../ch/errors" -import { makeExpr, type Condition, type Expr } from "../ch/expr" +import { QueryBuilderDefect, QueryBuilderError } from "../ch/errors" +import { makeCond, makeExpr, type Condition, type Expr } from "../ch/expr" import { schemaOf, withoutNull } from "../ch/define-fn" import { compile, known, str } from "../sql/sql-fragment" import { encodeLiteral } from "../ch/literal" +import { jsonPathSchema, type JsonContained, type JsonKey, type JsonPath, type JsonPathValue } from "../ch/json-path" import type { CHType } from "../ch/types" import { builtins } from "../ch/functions/builtin" import * as T from "./types" @@ -134,6 +135,133 @@ export const coalesce = (expr: Expr, fa export const jsonText = (expr: Expr, key: string): Expr => makeExpr(scalar.lazy(() => `(${sql(expr)} ->> ${compile(str(key))})`), Schema.NullOr(Schema.String) as Schema.Codec) +// jsonb paths +// +// The path's type and decoder come from the column's schema (src/ch/json-path.ts); +// only the SQL below is Postgres's. Keys are written as literals, not bound, so +// an expression index on `payload -> 'profile'` matches the query. + +const describePath = (path: ReadonlyArray): string => JSON.stringify(path) + +/** One key of a path as SQL: a string literal, or an array index. */ +const pathKey = (key: JsonKey, path: ReadonlyArray): string => + typeof key === "string" + ? compile(str(key)) + : Number.isSafeInteger(key) + ? String(key) + : fail(new QueryBuilderDefect({ message: `jsonPath ${describePath(path)}: an array index must be a whole number, got ${key}` }), "0") + +/** The keys as `-> k1 -> k2`, the last one with `last` (`->` or `->>`). */ +const arrows = (path: ReadonlyArray, last: "->" | "->>"): string => + path.length === 0 + ? fail(new QueryBuilderDefect({ message: "jsonPath: a path needs at least one key" }), "") + : path.map((key, i) => ` ${i === path.length - 1 ? last : "->"} ${pathKey(key, path)}`).join("") + +/** The schema at `path` inside `expr`'s document; a path the schema does not have is a defect when rendered. */ +const pathSchema = ( + expr: Expr, + path: ReadonlyArray, +): { readonly schema: Schema.Codec | undefined; readonly check: () => void } => { + if (expr.schema === undefined) return { schema: undefined, check: () => undefined } + const found = jsonPathSchema(expr.schema, path) + return found._tag === "Found" + ? { schema: found.schema, check: () => undefined } + : { + schema: Schema.Unknown as Schema.Codec, + check: () => fail(new QueryBuilderDefect({ message: `jsonPath ${describePath(path)}: the column's schema has no such path (${found.reason})` }), undefined), + } +} + +/** A value as JSON text through `schema`, bound, cast to jsonb. */ +const jsonValue = (schema: Schema.Codec, value: unknown, context: string): string => + value === undefined || value === null + ? fail(new QueryBuilderError({ code: "InvalidArguments", message: `${context}: no value given; write JSON null as a value of a nullable field's schema` }), "NULL") + : `${encodeLiteral(Schema.fromJsonString(schema), value, context)}::jsonb` + +/** + * `expr -> 'k1' -> 'k2'`: the jsonb value at a path, typed and decoded by the + * column's schema. `jsonPath($.payload, "profile", "city")` on a + * `jsonb(Schema.Struct({ profile: Schema.Struct({ city: Schema.String }) }))` + * column is an `Expr`; a number key indexes an array. `null` when + * a key is absent. Compared with a value, the value is encoded by the field's + * schema and bound as JSON: `.eq("Kyiv")` matches the JSON string `"Kyiv"`. + */ +export const jsonPath = , Q = never>( + expr: Expr, + ...path: P +): Expr, undefined> | null, Q> => { + const keys = path as unknown as ReadonlyArray + const { schema, check } = pathSchema(expr as Expr, keys) + const context = `jsonPath ${describePath(keys)}` + return makeExpr( + scalar.lazy(() => { + check() + return `(${sql(expr)}${arrows(keys, "->")})` + }, "jsonPath"), + schema === undefined ? undefined : (Schema.NullOr(schema) as Schema.Codec, undefined> | null, unknown>), + (value) => known(() => encodeLiteral(Schema.fromJsonString(schema ?? Schema.Unknown), value, context)), + ) +} + +/** + * `expr -> 'k1' ->> 'k2'`: the value at a path as text, `null` when a key is + * absent. A JSON string reads without its quotes; a number or object as its JSON + * text. For `like`, `ilike` and text comparisons. + */ +export const jsonPathText = , Q = never>(expr: Expr, ...path: P): Expr => { + const keys = path as unknown as ReadonlyArray + const { check } = pathSchema(expr as Expr, keys) + return makeExpr( + scalar.lazy(() => { + check() + return `(${sql(expr)}${arrows(keys, "->>")})` + }, "jsonPathText"), + Schema.NullOr(Schema.String) as Schema.Codec, + (value) => known(() => encodeLiteral(Schema.String, value, `jsonPathText ${describePath(keys)}`)), + ) +} + +/** + * `expr @> value::jsonb`: whether the document contains `value`, at any depth. + * Objects match on the keys given, arrays on the elements given (in any order). + * The value is bound as JSON text, as given: write a transformed field in its + * stored form. A GIN index on the column serves it. + */ +export const jsonContains = (expr: Expr, value: JsonContained): Condition => + makeCond( + scalar.lazy( + () => `(${sql(expr)} @> ${jsonValue(Schema.Unknown as Schema.Codec, value, "jsonContains")})`, + "jsonContains", + ), + ) + +/** + * `jsonb_set(expr, ARRAY['k1', 'k2'], value::jsonb)`: the document with one + * path replaced, to write in `set` without reading the row first. The value is + * encoded by the field's schema. The last key is created if it is absent; a + * missing parent leaves the document unchanged, and a NULL document stays NULL. + * Nest calls to set several paths. + */ +export const jsonSet = , Q = never>( + expr: Expr, + path: P, + value: Exclude, undefined>, +): Expr => { + const keys = path as unknown as ReadonlyArray + const { schema, check } = pathSchema(expr as Expr, keys) + const context = `jsonSet ${describePath(keys)}` + return makeExpr( + scalar.lazy(() => { + check() + if (keys.length === 0) return fail(new QueryBuilderDefect({ message: "jsonSet: a path needs at least one key" }), "NULL") + // jsonb_set takes the path as text[], an array index too. + const text = keys.map((key) => (typeof key === "number" ? `${pathKey(key, keys)}::text` : pathKey(key, keys))) + return `jsonb_set(${sql(expr)}, ARRAY[${text.join(", ")}], ${jsonValue(schema ?? (Schema.Unknown as Schema.Codec), value, context)})` + }, "jsonSet"), + expr.schema, + ) +} + /** `greatest(a, b, ...)`, decoding as `first` does. Postgres skips NULL arguments. */ export const greatest = (first: Expr, ...rest: ReadonlyArray>): Expr => makeExpr(scalar.lazy(() => `greatest(${[first, ...rest].map(sql).join(", ")})`), schemaOf(first)) diff --git a/src/pg/json.test-d.ts b/src/pg/json.test-d.ts new file mode 100644 index 0000000..41a21fe --- /dev/null +++ b/src/pg/json.test-d.ts @@ -0,0 +1,57 @@ +// A jsonb path's keys and result type come from the column's schema. + +import { expectTypeOf } from "expect-type" +import { Schema } from "effect" +import type { Expr } from "../ch/expr" +import * as PG from "../postgres" + +const Payload = Schema.Struct({ + profile: Schema.Struct({ city: Schema.String, zip: Schema.optionalKey(Schema.String), visits: Schema.NumberFromString }), + tags: Schema.Array(Schema.String), + archived: Schema.optional(Schema.NullOr(Schema.Struct({ city: Schema.String }))), +}) + +const Docs = PG.table("docs", { + columns: { id: PG.int4, payload: PG.jsonb(Payload), maybe: PG.nullable(PG.jsonb(Payload)), loose: PG.jsonb() }, + primaryKey: ["id"], +}) + +PG.from(Docs).select(($) => { + // Reads: the field's decoded type, or null when a key is absent (never undefined). + expectTypeOf(PG.jsonPath($.payload, "profile", "city")).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.payload, "profile", "visits")).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.payload, "profile", "zip")).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.payload, "tags", 0)).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.payload, "archived", "city")).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.maybe, "profile", "city")).toEqualTypeOf>() + expectTypeOf(PG.jsonPath($.loose, "any", 1, "key")).toEqualTypeOf>() + expectTypeOf(PG.jsonPathText($.payload, "profile", "visits")).toEqualTypeOf>() + + // Only paths the schema has. + // @ts-expect-error -- no such field + PG.jsonPath($.payload, "profile", "town") + // @ts-expect-error -- an array takes an index, not a key + PG.jsonPath($.payload, "tags", "first") + // @ts-expect-error -- a path needs a key + PG.jsonPath($.payload) + + // Comparisons take the field's type. + PG.jsonPath($.payload, "profile", "visits").eq(3) + // @ts-expect-error -- visits is a number + PG.jsonPath($.payload, "profile", "visits").eq("3") + + // Containment takes part of the document. + PG.jsonContains($.payload, { profile: { city: "Kyiv" }, tags: ["a"] }) + // @ts-expect-error -- city is a string + PG.jsonContains($.payload, { profile: { city: 1 } }) + + // jsonb_set: the value has the path's type, and the result is the document. + expectTypeOf(PG.jsonSet($.payload, ["profile", "visits"], 4)).toEqualTypeOf>() + // @ts-expect-error -- visits is a number + PG.jsonSet($.payload, ["profile", "visits"], "4") + // @ts-expect-error -- not a path of the document + PG.jsonSet($.payload, ["profile", "town"], "x") + return { id: $.id } +}) + +PG.update(Docs).set(($) => ({ payload: PG.jsonSet($.payload, ["profile", "city"], "Lviv") })) diff --git a/src/pg/json.test.ts b/src/pg/json.test.ts new file mode 100644 index 0000000..c6fc957 --- /dev/null +++ b/src/pg/json.test.ts @@ -0,0 +1,183 @@ +// Typed JSON paths on a jsonb column, against PGlite: each query is compiled by +// the builder and run, so a test passes only if Postgres accepts the SQL and the +// rows decode through the schema the path reads off the column. + +import { PGlite } from "@electric-sql/pglite" +import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest" +import { Effect, Schema } from "effect" +import type { CompiledQuery } from "../ch/compile" +import { clickhouseDialect } from "../ch/dialect" +import * as PG from "../postgres" + +const db = new PGlite() + +const Profile = Schema.Struct({ + city: Schema.String, + zip: Schema.optionalKey(Schema.String), + visits: Schema.NumberFromString, +}) + +const Payload = Schema.Struct({ + profile: Profile, + tags: Schema.Array(Schema.String), + scores: Schema.Record(Schema.String, Schema.Number), + archived: Schema.optional(Schema.NullOr(Profile)), +}) + +const docs = PG.table("docs", { + columns: { id: PG.int4, org: PG.text, payload: PG.jsonb(Payload), extra: PG.nullable(PG.jsonb()) }, + primaryKey: ["id"], +}) + +beforeAll(async () => { + await db.exec(` + CREATE TABLE docs (id int4 PRIMARY KEY, org text, payload jsonb NOT NULL, extra jsonb); + INSERT INTO docs VALUES + (1, 'o', '{"profile": {"city": "Kyiv", "zip": "01001", "visits": "3"}, "tags": ["a", "b"], "scores": {"x": 1}}', '{"k": 1}'), + (2, 'o', '{"profile": {"city": "Lviv", "visits": "10"}, "tags": [], "scores": {}}', NULL); + `) +}) + +afterAll(async () => { + await db.close() +}) + +const run = async (compiled: CompiledQuery): Promise> => { + const result = await db.query>(compiled.sql, [...compiled.parameters]) + return Effect.runPromise(compiled.decodeRows(result.rows)) +} + +describe("jsonb paths", () => { + it("reads a nested field with ->, decoded by the field's schema", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select(($) => ({ id: $.id, city: PG.jsonPath($.payload, "profile", "city"), visits: PG.jsonPath($.payload, "profile", "visits") })) + .orderBy(["id", "asc"]), + {}, + ) + expect(compiled.sql).toContain(`("docs"."payload" -> 'profile' -> 'city') AS "city"`) + expect(compiled.rowSchemaSource).toBe("derived") + expect(await run(compiled)).toEqual([ + { id: 1, city: "Kyiv", visits: 3 }, + { id: 2, city: "Lviv", visits: 10 }, + ]) + }) +}) + +describe("jsonb paths: missing values", () => { + it("reads an absent key, an array element and a record value as null when absent", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select(($) => ({ + zip: PG.jsonPath($.payload, "profile", "zip"), + firstTag: PG.jsonPath($.payload, "tags", 0), + x: PG.jsonPath($.payload, "scores", "x"), + archivedCity: PG.jsonPath($.payload, "archived", "city"), + })) + .orderBy(($) => [[$.id, "asc"]]), + {}, + ) + expect(compiled.sql).toContain(`("docs"."payload" -> 'tags' -> 0)`) + expect(await run(compiled)).toEqual([ + { zip: "01001", firstTag: "a", x: 1, archivedCity: null }, + { zip: null, firstTag: null, x: null, archivedCity: null }, + ]) + }) +}) + +describe("jsonb path filters", () => { + it("compares a path with a value encoded by the field's schema and bound as JSON", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select("id") + .where(($) => [PG.jsonPath($.payload, "profile", "city").eq("Lviv"), PG.jsonPath($.payload, "profile", "visits").eq(10)]), + {}, + ) + expect(compiled.sql).toContain(`("docs"."payload" -> 'profile' -> 'city') = $1`) + // `visits` is a NumberFromString: 10 is stored, and compared, as "10". + expect(compiled.parameters).toEqual(['"Lviv"', '"10"']) + expect(await run(compiled)).toEqual([{ id: 2 }]) + }) + + it("reads a path as text with ->> for LIKE", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select(($) => ({ id: $.id, city: PG.jsonPathText($.payload, "profile", "city") })) + .where(($) => [PG.jsonPathText($.payload, "profile", "city").like("K%")]), + {}, + ) + expect(compiled.sql).toContain(`("docs"."payload" -> 'profile' ->> 'city')`) + expect(await run(compiled)).toEqual([{ id: 1, city: "Kyiv" }]) + }) + + it("filters with @> on the document or on a path", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select("id") + .where(($) => [PG.jsonContains($.payload, { profile: { city: "Kyiv" }, tags: ["b"] }), PG.jsonContains(PG.jsonPath($.payload, "tags"), ["a"])]), + {}, + ) + expect(compiled.sql).toContain(`("docs"."payload" @> $1::jsonb)`) + expect(compiled.parameters).toEqual(['{"profile":{"city":"Kyiv"},"tags":["b"]}', '["a"]']) + expect(await run(compiled)).toEqual([{ id: 1 }]) + }) + + it("takes any keys on a jsonb column with no schema, and reads unknown", async () => { + const compiled = PG.compileUnsafe( + PG.from(docs) + .select(($) => ({ id: $.id, k: PG.jsonPath($.extra, "k") })) + .orderBy(["id", "asc"]), + {}, + ) + expect(await run(compiled)).toEqual([ + { id: 1, k: 1 }, + { id: 2, k: null }, + ]) + }) +}) + +describe("jsonb path failures", () => { + it("is a defect for a path the schema does not have, and for a ClickHouse compile", () => { + // @ts-expect-error -- not a field of the profile + const wrong = PG.from(docs).select(($) => ({ v: PG.jsonPath($.payload, "profile", "town") })) + expect(() => PG.compileUnsafe(wrong, {})).toThrow(/jsonPath \["profile","town"\]: the column's schema has no such path \(at \["profile","town"\]: no field "town"\)/) + + const ok = PG.from(docs).select(($) => ({ v: PG.jsonPath($.payload, "profile", "city") })) + expect(() => PG.compileUnsafe(ok, {}, { dialect: clickhouseDialect })).toThrow(/jsonPath\(\) is a Postgres function/) + }) + + it("refuses a path through a field whose stored form is not its type", () => { + const Renamed = Schema.Struct({ inner: Schema.Struct({ a: Schema.String }).pipe(Schema.encodeKeys({ a: "A" })) }) + const t = PG.table("r", { columns: { id: PG.int4, doc: PG.jsonb(Renamed) }, primaryKey: ["id"] }) + const query = PG.from(t).select(($) => ({ v: PG.jsonPath($.doc, "inner", "a") })) + expect(() => PG.compileUnsafe(query, {})).toThrow(/its stored form is a transformation of its type/) + }) +}) + +describe("jsonb values that are not JSON", () => { + it("fails the compile with InvalidLiteral instead of throwing", () => { + const query = PG.from(docs) + .select("id") + .where(($) => [PG.jsonContains($.extra, { n: 1n })]) + expect(() => PG.compileUnsafe(query, {})).toThrow(/jsonContains: an object is not a valid value/) + }) +}) + +describe("jsonb_set", () => { + it("sets one path in an UPDATE, encoded by the field's schema, and returns the decoded document", async () => { + const compiled = PG.compileUnsafe( + PG.update(docs) + .set(($) => ({ payload: PG.jsonSet(PG.jsonSet($.payload, ["profile", "visits"], 11), ["tags", 0], "z") })) + .where(($) => [$.id.eq(1)]) + .returning(($) => ({ payload: $.payload })), + {}, + ) + expect(compiled.sql).toBe( + `UPDATE "docs" SET "payload" = jsonb_set(jsonb_set("payload", ARRAY['profile', 'visits'], $1::jsonb), ARRAY['tags', 0::text], $2::jsonb)\nWHERE "id" = $3\nRETURNING "payload" AS "payload"`, + ) + expect(compiled.parameters).toEqual(['"11"', '"z"', 1]) + const [row] = await run(compiled) + expect(row?.payload.profile).toEqual({ city: "Kyiv", zip: "01001", visits: 11 }) + expect(row?.payload.tags).toEqual(["z", "b"]) + }) +}) From 9e8fa2ceb376613f7457f304f67c44b43616045b Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:37:37 +0200 Subject: [PATCH 2/4] docs(postgres): document typed jsonb paths and how other dialects reuse them --- .changeset/pg-json-paths.md | 5 +++ design/dialects.md | 8 ++++ design/gap-review.md | 2 +- docs/postgres.md | 79 ++++++++++++++++++++++++++++++++++ docs/reference.md | 5 ++- scripts/check-doc-examples.mjs | 6 +++ src/postgres.ts | 11 +++++ 7 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 .changeset/pg-json-paths.md diff --git a/.changeset/pg-json-paths.md b/.changeset/pg-json-paths.md new file mode 100644 index 0000000..4f00971 --- /dev/null +++ b/.changeset/pg-json-paths.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Typed jsonb paths on Postgres, read off the column's schema. `PG.jsonPath($.payload, "profile", "city")` on a `PG.jsonb(Schema.Struct({ profile: Schema.Struct({ city: Schema.String }) }))` column is an `Expr` that compiles to `("payload" -> 'profile' -> 'city')` and decodes with the field's own schema; only paths the schema has type-check, and a number key indexes an array. `jsonPathText` reads with `->>`, `jsonContains` filters with `@>` (bound as `$n::jsonb`), and `jsonSet` writes one path with `jsonb_set` inside an UPDATE's `set`, without reading the row first. Values compared with a path, and the value of a `jsonSet`, are encoded by the field's schema and bound as JSON. A path the schema does not have, or one through a field whose stored form differs from its type, is a defect at compile time. The path types (`JsonPath`, `JsonPathValue`, `JsonContained`, …) and `jsonPathSchema` are exported for helpers. diff --git a/design/dialects.md b/design/dialects.md index b02cba5..66607c0 100644 --- a/design/dialects.md +++ b/design/dialects.md @@ -84,5 +84,13 @@ What we deliberately do not copy: decision for a later release. - The ClickHouse function catalog writes ClickHouse SQL whatever the dialect. Functions could refuse to compile under another dialect instead of producing SQL that fails at the server. +- JSON paths split the same way. `src/ch/json-path.ts` owns what no dialect changes: the path + types (`JsonPath`, `JsonPathValue`, `JsonContained`) and `jsonPathSchema`, which reads the + decoder at a path off the column's Schema. `src/pg/functions.ts` owns only the SQL (`->`, + `->>`, `@>`, `jsonb_set`). A SQLite or MySQL catalog adds its own `jsonPath`, `jsonSet` and + friends over the same keys: `json_extract(x, '$.a[0]')` / `x ->> '$.a'` and + `json_set(x, '$.a', json(?))` on SQLite; `JSON_EXTRACT`, `JSON_UNQUOTE`, + `JSON_CONTAINS(x, CAST(? AS JSON))` and `JSON_SET` on MySQL. The one new piece is a key list + written as a `$.a[0]` path string, with each key quoted (`$."a b"`). - MySQL or SQLite would need `?` placeholders (`reuse: false` already exists) and their own types and functions; nothing in the core should need to change. diff --git a/design/gap-review.md b/design/gap-review.md index 852c5eb..80eebed 100644 --- a/design/gap-review.md +++ b/design/gap-review.md @@ -30,7 +30,7 @@ builder; **P1** commonly used; **P2** niche. | Postgres column types: `timestamptz` as `Date`, `timestamp`, `date`, `interval`, `varchar(n)`, serial / identity | 226 timestamp columns | S | | ~~DISTINCT (and DISTINCT ON)~~ (built) | ~10 | S | | ~~`FOR UPDATE` / `FOR SHARE` / `SKIP LOCKED` / `NOWAIT`~~ (built) | 7 | S | -| jsonb and array operators (`@>`, `->`, `?`, `&&`, `ANY`) | ~12 | M | +| jsonb and array operators (`@>`, `->`, `?`, `&&`, `ANY`); built: typed `->` / `->>` paths, `@>`, `jsonb_set` (docs/postgres.md#json-paths); open: `?`, `&&`, `ANY` | ~12 | M | | ~~`isNull` / `isNotNull` / `between`; variadic `and` / `or` that skip `undefined`~~ (built) | everywhere | S | | Constraint error helpers (unique, foreign key, not null); keep ClickHouse's numeric error codes, which `sqlStateOf` drops today | all upserts | S | | Tenant-scope enforcement in `Database`, opt in, with an explicit cross-tenant entry point | safety | S | diff --git a/docs/postgres.md b/docs/postgres.md index 8964dcb..118986e 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -182,6 +182,84 @@ either form, and writes its value as array text, which Postgres casts to the col work for an enum `generate` does not manage; see [migrations](./migrations.md#postgres) for moving one to `enumType`. +## JSON paths + +A `jsonb(schema)` column knows the shape of its documents, so a path into one is typed and +decoded by the same schema. `jsonPath(column, ...keys)` reads the value at a path with `->`; +a string key steps into an object, a number into an array. Only paths the schema has +type-check, and the result decodes with the field's own schema (a `NumberFromString` field +reads as a number). A key that is absent reads as `null`. + +```ts title="postgres-json-paths.ts" +import { PGlite } from "@electric-sql/pglite" +import { Effect, Schema } from "effect" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Settings = Schema.Struct({ + notifications: Schema.Struct({ sms: Schema.Boolean, email: Schema.Boolean }), + tags: Schema.Array(Schema.String), +}) + +const Users = PG.table("users", { + columns: { id: PG.int4, settings: PG.jsonb(Settings) }, + primaryKey: ["id"], +}) + +// Read and filter: `sms` is an Expr. +export const read = PG.compileUnsafe( + PG.from(Users) + .select(($) => ({ id: $.id, sms: PG.jsonPath($.settings, "notifications", "sms") })) + .where(($) => [PG.jsonPath($.settings, "notifications", "email").eq(true), PG.jsonContains($.settings, { tags: ["beta"] })]), + {}, +) +// read.sql: SELECT "users"."id" AS "id", ("users"."settings" -> 'notifications' -> 'sms') AS "sms" FROM "users" +// WHERE ("users"."settings" -> 'notifications' -> 'email') = $1 AND ("users"."settings" @> $2::jsonb) +// read.parameters: ["true", '{"tags":["beta"]}'] + +// Write one path, without reading the row first. +export const write = PG.compileUnsafe( + PG.update(Users) + .set(($) => ({ settings: PG.jsonSet($.settings, ["notifications", "sms"], false) })) + .where(($) => [$.id.eq(1)]), + {}, +) +// write.sql: UPDATE "users" SET "settings" = jsonb_set("settings", ARRAY['notifications', 'sms'], $1::jsonb) WHERE "id" = $2 + +const db = new PGlite() +await db.exec(` + CREATE TABLE users (id int4 PRIMARY KEY, settings jsonb NOT NULL); + INSERT INTO users VALUES (1, '{"notifications": {"sms": true, "email": true}, "tags": ["beta"]}'); +`) +await db.query(write.sql, [...write.parameters]) +const result = await db.query>(read.sql, [...read.parameters]) +export const rows = await Effect.runPromise(read.decodeRows(result.rows)) +// [{ id: 1, sms: false }] +await db.close() +``` + +| Function | SQL | Result | +| --- | --- | --- | +| `jsonPath(x, ...keys)` | `(x -> 'k1' -> 'k2')` | the field's type, or `null` | +| `jsonPathText(x, ...keys)` | `(x -> 'k1' ->> 'k2')` | `string \| null`: a JSON string without its quotes | +| `jsonContains(x, part)` | `(x @> $n::jsonb)` | a condition | +| `jsonSet(x, [keys], value)` | `jsonb_set(x, ARRAY['k1', 'k2'], $n::jsonb)` | the document's type | + +- A value compared with a `jsonPath`, and the value of a `jsonSet`, is encoded by the field's + schema and bound as JSON text: `.eq("Kyiv")` matches the JSON string `"Kyiv"`. For text + comparisons and `like`, use `jsonPathText`. +- `jsonContains` takes part of the document, at any depth: objects match on the keys given, + arrays on the elements given, in any order. It accepts a `jsonPath` too. Its value is written + as given, so a field whose schema transforms (a date stored as text) is written in its stored + form. A GIN index on the column serves `@>`. +- `jsonSet` creates the last key if it is absent. A missing parent leaves the document as it is, + and a `NULL` document stays `NULL`. Nest calls to set more than one path. +- Keys are written into the SQL as literals, not bound, so an expression index on + `(settings -> 'notifications')` matches. +- The schema walk follows structs, records, arrays and tuples, through unions, `NullOr`, + optional keys and `Schema.suspend`. A path through a field whose stored form is not its + type (`Schema.encodeKeys`, a class) is refused when the query compiles. A `jsonb()` column + with no schema takes any keys and reads `unknown`. + ## Functions | Function | SQL | Notes | @@ -197,6 +275,7 @@ moving one to `enumType`. | `lower`, `upper`, `length` | same | | | `coalesce(x, fallback)` | `coalesce(x, fallback)` | No longer nullable | | `jsonText(x, key)` | `(x ->> key)` | `null` when absent | +| `jsonPath`, `jsonPathText`, `jsonContains`, `jsonSet` | `->`, `->>`, `@>`, `jsonb_set` | Typed by the column schema; see [JSON paths](#json-paths) | | `greatest(a, ...)`, `least(a, ...)` | same | NULL arguments are skipped | | `caseWhen([[c, v], ...], otherwise)` | `CASE WHEN c THEN v ... ELSE otherwise END` | | | `asBoolean(c)` | `(c)` | A condition as a value, to select or `set` | diff --git a/docs/reference.md b/docs/reference.md index abc2d46..d765ec2 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -489,9 +489,12 @@ one-dimensional array literal (`{a,"b c",NULL}`), the form enum arrays arrive in `max(x)`, `percentileCont(f, x)`, `arrayAgg(x)`, `dateTrunc(unit, ts)` (`DateTruncUnit` is the unit union), `dateBin(seconds, ts)`, `now()`, `lower(x)`, `upper(x)`, `length(x)`, `coalesce(x, fallback)`, `nullIf(x, value)`, `jsonText(x, key)`, `greatest(a, ...)`, `least(a, ...)`, +`jsonPath(x, ...keys)`, `jsonPathText(x, ...keys)`, `jsonContains(x, part)`, `jsonSet(x, keys, value)` +(typed jsonb paths; the types `JsonKey`, `JsonKeyOf`, `JsonStep`, `JsonPath`, `JsonPathValue` and +`JsonContained` spell them, and `jsonPathSchema` reads a path's schema off a document schema, as a `JsonPathSchema`), `caseWhen([[c, v], ...], otherwise)`, `asBoolean(c)` (a condition as a value) and `typedValue(type, value)` (a value bound as a column type writes it). See -[Postgres functions](./postgres.md#functions) for the SQL each writes. +[Postgres functions](./postgres.md#functions) and [JSON paths](./postgres.md#json-paths) for the SQL each writes. ### Dialect diff --git a/scripts/check-doc-examples.mjs b/scripts/check-doc-examples.mjs index d8b190a..0e2a9af 100644 --- a/scripts/check-doc-examples.mjs +++ b/scripts/check-doc-examples.mjs @@ -117,6 +117,12 @@ assert.deepEqual(enums.shared.parameters, ["public", "private"]) assert.deepEqual(await Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "public" }])), [{ id: "c", type: "public" }]) await assert.rejects(() => Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "secret" }]))) assert.match(sql(enums.upsert), /ON CONFLICT \\("slug"\\) DO UPDATE SET/) +const json = await import("./postgres-json-paths") +assert.ok(sql(json.read).includes("(\\"users\\".\\"settings\\" -> 'notifications' -> 'sms') AS \\"sms\\"")) +assert.ok(sql(json.read).includes("(\\"users\\".\\"settings\\" @> $2::jsonb)")) +assert.deepEqual(json.read.parameters, ["true", '{"tags":["beta"]}']) +assert.equal(sql(json.write), "UPDATE \\"users\\" SET \\"settings\\" = jsonb_set(\\"settings\\", ARRAY['notifications', 'sms'], $1::jsonb) WHERE \\"id\\" = $2") +assert.deepEqual(json.rows, [{ id: 1, sms: false }]) const database = await import("./database-transaction") assert.deepEqual(database.balances, { from: 70, to: 30, refused: "InsufficientFunds" }) console.log("Markdown example behavior checks passed") diff --git a/src/postgres.ts b/src/postgres.ts index 1610096..0d4cd9a 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -19,6 +19,17 @@ export * from "./pg/functions" // Portable: renders the same SQL on both dialects. export { nullIf } from "./ch/functions" export { postgresDialect } +// Typed JSON paths: the keys and decoder a jsonb path reads off the column's schema. +export { + jsonPathSchema, + type JsonContained, + type JsonKey, + type JsonKeyOf, + type JsonPath, + type JsonPathSchema, + type JsonPathValue, + type JsonStep, +} from "./ch/json-path" // Tables and their DDL. `table` is the only way to declare a table: it carries // the keys, indexes, unique constraints and foreign keys `effect-orm generate` From dc565d0360f018fc258333ac29ae670f8ede1a5e Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:38:44 +0200 Subject: [PATCH 3/4] test(postgres): cover the jsonb path exports in the dialect manifest --- tests/dialect-cases.postgres.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/dialect-cases.postgres.ts b/tests/dialect-cases.postgres.ts index b502fac..27b4ae7 100644 --- a/tests/dialect-cases.postgres.ts +++ b/tests/dialect-cases.postgres.ts @@ -281,6 +281,25 @@ export const postgresCases: readonly PostgresCase[] = [ ), expected: [{ region: "eu", absent: null }], }, + { + id: "json-paths", + covers: pg("jsonPath", "jsonPathText", "jsonContains", "jsonSet", "jsonPathSchema"), + build: () => { + const found = PG.jsonPathSchema(Schema.Struct({ region: Schema.String }), ["region"]) + expect(found._tag).toBe("Found") + return PG.compileUnsafe( + typedRows() + .select(($) => ({ + region: PG.jsonPath($.Doc, "region"), + text: PG.jsonPathText($.Doc, "region"), + moved: PG.jsonPath(PG.jsonSet($.Doc, ["region"], "us"), "region"), + })) + .where(($) => [PG.jsonPath($.Doc, "region").eq("eu"), PG.jsonContains($.Doc, { region: "eu" })]), + {}, + ) + }, + expected: [{ region: "eu", text: "eu", moved: "us" }], + }, { id: "conditionals-and-typed-values", covers: pg("greatest", "least", "caseWhen", "asBoolean", "typedValue"), From 88fa4e6c0f4ba3fe966aea1aca4d1d2508647b36 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:41:37 +0200 Subject: [PATCH 4/4] fix(postgres): jsonSet writes JSON null where the field's schema takes it --- src/pg/functions.ts | 6 +++--- src/pg/json.test.ts | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/pg/functions.ts b/src/pg/functions.ts index a8cc709..109e845 100644 --- a/src/pg/functions.ts +++ b/src/pg/functions.ts @@ -172,10 +172,10 @@ const pathSchema = ( } } -/** A value as JSON text through `schema`, bound, cast to jsonb. */ +/** A value as JSON text through `schema`, bound, cast to jsonb. `null` is JSON null; `undefined` has no JSON. */ const jsonValue = (schema: Schema.Codec, value: unknown, context: string): string => - value === undefined || value === null - ? fail(new QueryBuilderError({ code: "InvalidArguments", message: `${context}: no value given; write JSON null as a value of a nullable field's schema` }), "NULL") + value === undefined + ? fail(new QueryBuilderError({ code: "InvalidArguments", message: `${context}: undefined has no JSON form` }), "NULL") : `${encodeLiteral(Schema.fromJsonString(schema), value, context)}::jsonb` /** diff --git a/src/pg/json.test.ts b/src/pg/json.test.ts index c6fc957..16dc575 100644 --- a/src/pg/json.test.ts +++ b/src/pg/json.test.ts @@ -180,4 +180,22 @@ describe("jsonb_set", () => { expect(row?.payload.profile).toEqual({ city: "Kyiv", zip: "01001", visits: 11 }) expect(row?.payload.tags).toEqual(["z", "b"]) }) + + it("writes JSON null where the field's schema takes it, and refuses it where it does not", async () => { + const compiled = PG.compileUnsafe( + PG.update(docs) + .set(($) => ({ payload: PG.jsonSet($.payload, ["archived"], null) })) + .where(($) => [$.id.eq(2)]) + .returning(($) => ({ archived: PG.jsonPath($.payload, "archived") })), + {}, + ) + expect(compiled.parameters).toEqual(["null", 2]) + expect(await run(compiled)).toEqual([{ archived: null }]) + + const refused = PG.update(docs) + // @ts-expect-error -- city is not nullable + .set(($) => ({ payload: PG.jsonSet($.payload, ["profile", "city"], null) })) + .where(($) => [$.id.eq(2)]) + expect(() => PG.compileUnsafe(refused, {})).toThrow(/jsonSet \["profile","city"\]: null is not a valid value/) + }) })