diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md index 32eaa74..9f58d58 100644 --- a/.changeset/quiet-sqlite-dialects.md +++ b/.changeset/quiet-sqlite-dialects.md @@ -7,5 +7,4 @@ Add the `/sqlite` query entry with SQLite codecs, functions and positional bindi and explicit refusal of interactive transactions. `timestamp` stores SQLite's own `YYYY-MM-DD HH:MM:SS.SSS` UTC text, so it compares correctly with what `strftime` writes; `LIKE` escapes with `\` as on every dialect; text with U+0000 and integers past 2^53 are refused rather -than silently changed. `table` is a query schema only: SQLite DDL and -migration generation are not implemented yet. +than silently changed. SQLite migration generation is not implemented yet. diff --git a/.changeset/sqlite-table-ddl.md b/.changeset/sqlite-table-ddl.md new file mode 100644 index 0000000..629d3ec --- /dev/null +++ b/.changeset/sqlite-table-ddl.md @@ -0,0 +1,11 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`SQLite.table(name, { columns, primaryKey, indexes })` declares a SQLite table with its DDL: +`NOT NULL` from the column type, single and composite primary keys, literal and expression +defaults (`SQLite.timestampNow` for a `timestamp` column), unique columns, and named indexes and +unique indexes. `sqliteEntitiesOf` and +`renderSqliteSchema` from `/schema` check a schema and write its `CREATE TABLE` and +`CREATE INDEX` statements. `table(name, columns, options?)` and +`table(name, { external: true, columns })` stay query-only. SQLite migrations are not generated yet. diff --git a/README.md b/README.md index 4ad708a..aef1e51 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproduc Formerly `@maple-dev/effect-clickhouse`. SQLite and Cloudflare D1 use the same builder and row decoder through the new -[`/sqlite` entry](./docs/sqlite.md). SQLite schema/migration generation is not implemented. +[`/sqlite` entry](./docs/sqlite.md), which also writes SQLite table DDL. SQLite migration generation is not implemented. [Read the documentation](https://effect-clickhouse.maple.dev) · [Getting started](./docs/getting-started.md) · [Recipes](./docs/recipes.md) diff --git a/docs/README.md b/docs/README.md index cbdb224..e9499a4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -102,4 +102,4 @@ See [Benchmarking](./benchmarking.md) and the ## SQLite and D1 -[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, Effect SQL drivers, tenant checks and transaction limits. SQLite migration generation is not implemented. +[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, table DDL, Effect SQL drivers, tenant checks and transaction limits. SQLite migration generation is not implemented. diff --git a/docs/reference.md b/docs/reference.md index f7cd842..5b81003 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -241,7 +241,7 @@ Tag `"@maple-dev/effect-orm/CompiledQueryDecodeError"`. Fails the `decodeRows` / #### `SchemaDefinitionError` -Tag `"@maple-dev/effect-orm/SchemaDefinitionError"`. Fails `S.entitiesOf`, `S.pgEntitiesOf` and +Tag `"@maple-dev/effect-orm/SchemaDefinitionError"`. Fails `S.entitiesOf`, `S.pgEntitiesOf`, `S.sqliteEntitiesOf` and `TB.buildProject` when a definition cannot render: a name that is not a plain identifier, a MergeTree-family engine with no `orderBy`, a view reading from a union, a view writing to a table outside the schema. Fields: `problems` (every `DefinitionProblem`, `{ object, message }`) and @@ -510,7 +510,7 @@ placeholders, double-quoted identifiers. | `SqlQuery`, `compileQuery` | `/sql` | | `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | | `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | -| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint` | `/schema` | +| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint`; for SQLite, `sqliteEntitiesOf`, `renderSqliteSchema`, `renderSqliteCreateTable`, `renderSqliteCreateIndex` and the `SqliteTableEntity`, `SqliteColumnEntity`, `SqliteIndexEntity`, `SqliteUniqueEntity` schemas | `/schema` | | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | @@ -522,8 +522,8 @@ placeholders, double-quoted identifiers. | `makeHttpClient`, `makeHttpTransport`, `httpConfigFromEnv` | `/benchmark/http` | | `runCli` | `/benchmark/cli` | -`/schema` reads tables; it does not declare them. Declare tables with `table` from `/clickhouse` -or `/postgres`. See [Schema and migrations](./migrations.md), [Database](./database.md), +`/schema` reads tables; it does not declare them. Declare tables with `table` from `/clickhouse`, +`/postgres` or `/sqlite`. See [Schema and migrations](./migrations.md), [Database](./database.md), [Running a query](./running-queries.md) for what the `/sql` statement helpers are for, and [Benchmarking](./benchmarking.md) for the complete benchmark API, command workflow, result verification, and JSON protocol. @@ -545,13 +545,54 @@ interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and | `boolean` | A boolean encoded as 0/1; other integers fail decoding | | `timestamp` | An instant stored as SQLite's `YYYY-MM-DD HH:MM:SS.SSS` UTC text; reads ISO-8601 too | | `sqliteTimestampText(epochMillis)` | That text for an instant, for raw SQL and seeds | +| `timestampNow` | A `defaultExpr` for a `timestamp` column: the current time in that text form | | `blob` | Bytes from a Uint8Array or D1's array wire representation | | `json(schema)` | JSON text decoded through the supplied schema | | `nullable(type)`, `custom`, `brand` | Type composition | -| `table(name, columns, options?)`, `TableOptions` | A query schema, including `tenantColumn`, defaults and computed-column metadata; no generated DDL | -| `table(name, ExternalTableDefinition)`, `ExternalTableDefinition` | `{ external: true, columns, tenantColumn? }`, the same query schema in the `/clickhouse` and `/postgres` form | | `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min`, `max` | SQLite aggregates; the last four can return SQL NULL | | `lower`, `upper`, `length`, `coalesce` | SQLite scalar functions | -SQLite's table definition is for query typing and decoding. The schema/kit/migrate entries -currently support ClickHouse and Postgres DDL only. +### Tables and DDL + +```ts +const Links = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.column(SQLite.text, { unique: true }), + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + clickCount: SQLite.column(SQLite.integer, { name: "click_count", default: 0 }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["id"], + indexes: [SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) +``` + +| Export | Signature | +| --- | --- | +| `table` | `(name, TableDefinition) => SqliteSchemaTable`, `(name, ExternalTableDefinition) => Table`, or `(name, columns, TableOptions?) => Table`, a query schema with no DDL | +| `column` | `(type, ColumnOptions) => ColumnSpec` | +| `index` | `(name, on) => IndexSpec`: columns, or a callback building expressions | +| `uniqueIndex` | The same, `UNIQUE` | + +**`TableDefinition`** — `columns` (required), `primaryKey` (column names: one is written inline, +several as a table constraint), `indexes`, `tenantColumn`. A column is `NOT NULL` unless its +type is `nullable(...)`; a primary key column cannot be nullable. + +**`ExternalTableDefinition`** — `{ external: true, columns, tenantColumn? }`: a view, or a table +another tool migrates. No DDL. **`TableOptions`** — `tenantColumn`, `defaults`, `computed`, for +the bare-columns form. + +**`ColumnOptions`** — one of `default` (a literal, encoded through the column type; a boolean +is written `true` / `false`) or `defaultExpr` (SQL as written, so add SQLite's parentheses, or a +DSL callback, parenthesized for you); either lets an insert leave the column out +(`DefaultedColumnsOf`). Also `name` (the column's name in SQL) and `unique` (`true` or a +constraint name), an inline `UNIQUE` constraint named `__unique` by default, as +drizzle-kit 1.0 writes and names it. + +**Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `TableDdl`, `SqliteSchemaTable`, +`DdlExpr`, `DdlKey`, `DefinitionProblem`. A definition that cannot render records a problem; +`S.sqliteEntitiesOf` fails with a `SchemaDefinitionError`, and `S.renderSqliteSchema` writes +the `CREATE TABLE` and `CREATE INDEX` statements. There is no SQLite diff, snapshot or migration +runner yet. diff --git a/docs/sqlite.md b/docs/sqlite.md index 05bc3fd..5743d09 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -24,9 +24,68 @@ export const compiled = SQLite.compileUnsafe(byOrg, { orgId: "org_1" }) // compiled.parameters is ["org_1"]; compiled.tenantScope is "single-tenant". ``` -`table` describes the query schema. Manage its actual DDL and migrations outside this package. -Unlike the ClickHouse and Postgres entries, SQLite has no schema diff, introspection, migration -ledger or kit dialect yet. A `table` value here cannot be passed to the DDL generators. +`table(name, columns, options?)` is a query schema only: it carries no DDL. + +## Declare a table with its DDL + +Pass `{ columns, primaryKey, indexes }` to `table` to declare the table's DDL too. The value is +still a query `Table`. `sqliteEntitiesOf` from `/schema` checks a set of tables as one schema, +and `renderSqliteSchema` writes their `CREATE TABLE` and `CREATE INDEX` statements: + +```ts title="sqlite-ddl.ts" +import { Effect } from "effect" +import * as S from "@maple-dev/effect-orm/schema" +import * as SQLite from "@maple-dev/effect-orm/sqlite" + +export const ShortLinks = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.text, + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + clickCount: SQLite.column(SQLite.integer, { name: "click_count", default: 0 }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) + +export const statements = S.sqliteEntitiesOf([ShortLinks]).pipe(Effect.map(S.renderSqliteSchema)) +// CREATE TABLE IF NOT EXISTS "short_link" ( +// "id" text PRIMARY KEY NOT NULL, +// "code" text NOT NULL, +// "org_id" text, +// "click_count" integer NOT NULL DEFAULT 0, +// "created_at" text NOT NULL DEFAULT (CURRENT_TIMESTAMP) +// ) +// CREATE UNIQUE INDEX IF NOT EXISTS "short_link_code" ON "short_link" ("code") +// CREATE INDEX IF NOT EXISTS "short_link_org" ON "short_link" ("org_id") +``` + +- A column is `NOT NULL` unless its type is `nullable(...)`. A primary key column cannot be + nullable: SQLite would accept NULL keys in it. +- One primary key column is written inline (`PRIMARY KEY`); several become a table constraint + `PRIMARY KEY ("a", "b")`. SQLite stores no constraint name, so there is none to give. +- `default` takes a literal of the column type. `defaultExpr` takes SQL as written, so add the + parentheses SQLite requires around an expression, or a DSL callback, which is parenthesized + for you. Either lets an insert leave the column out. +- `unique: true` writes an inline `UNIQUE` constraint, as drizzle-kit 1.0 does for `.unique()`. + Its name, `
__unique` or the string you give, is kept in the schema entities; + SQLite itself names the backing index `sqlite_autoindex_...`. +- Column types declare their SQLite type: `text`, `timestamp` and `json` are `text`; `integer` + and `boolean` are `integer`; `real` is `real`; `blob` is `blob`. A `custom(sql, schema)` type + declares `sql`, for example `numeric`. +- `table(name, { external: true, columns })` declares a table another tool owns: no DDL. + +The tests apply this DDL and drizzle-kit's SQL for the same tables to real SQLite, and compare +`PRAGMA table_info`, `index_list` and `index_xinfo`. One difference is deliberate: drizzle-kit +1.0 drops `NOT NULL` from a primary key column, so SQLite then accepts a NULL key in a `text` +key column. This entry keeps `NOT NULL`. + +There is no SQLite schema diff, introspection, migration ledger or kit dialect yet. Run the +rendered statements yourself, or keep migrating with your current tool. `WITHOUT ROWID`, +`STRICT`, `AUTOINCREMENT`, partial indexes, foreign keys and `CHECK` constraints are not +declared yet. The entry exports `text`, `integer` (a safe JavaScript integer), `real` (finite numbers), `boolean` (0/1), `blob`, `timestamp`, `json(schema)`, `nullable`, `custom` and `brand`. @@ -40,7 +99,8 @@ form `strftime('%Y-%m-%d %H:%M:%f')` and `datetime(..., 'subsec')` write. It rea and ISO-8601; text without a zone is UTC. SQLite compares text by bytes, so a timestamp column orders by time only when every value in it has one form: write them through the builder or with `strftime('%Y-%m-%d %H:%M:%f', ...)`. `CURRENT_TIMESTAMP` writes no milliseconds -(`2026-01-01 00:00:00`), which compares as earlier than the same instant written with them. +(`2026-01-01 00:00:00`), which compares as earlier than the same instant written with them, so +a `timestamp` column refuses it as a `defaultExpr`; use `defaultExpr: SQLite.timestampNow`. `blob` accepts the Uint8Array wire form from Node and the array-of-bytes form from D1. `json` always uses the supplied schema to validate decoded JSON. Use `text` when a stored date/time must remain a string. diff --git a/src/schema.ts b/src/schema.ts index 7fe9f4d..6d6910b 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -87,5 +87,21 @@ export { type PgOpLabel, } from "./schema/pg-ops" export { diffPgSchemas } from "./schema/pg-diff" +export { sqliteEntitiesOf, type SqliteSchemaTable } from "./schema/sqlite-define" +export { + SqliteColumnEntity, + SqliteIndexEntity, + SqliteSchemaEntity, + SqliteTableEntity, + SqliteUniqueEntity, + canonicalSqliteType, +} from "./schema/sqlite-entities" +export { + renderSqliteColumnDefinition, + renderSqliteCreateIndex, + renderSqliteCreateTable, + renderSqliteSchema, + sqliteIdent, +} from "./schema/sqlite-render" export { fromDrizzleSnapshot, type DrizzleImport } from "./schema/drizzle" export { Hint, Hints, diffSchemas, type DiffResult, type UnsupportedChange } from "./schema/diff" diff --git a/src/schema/ddl-expr.ts b/src/schema/ddl-expr.ts new file mode 100644 index 0000000..a8afc6e --- /dev/null +++ b/src/schema/ddl-expr.ts @@ -0,0 +1,34 @@ +// Expressions in DDL: defaults, index keys and predicates, written with the +// query DSL or as SQL strings, and rendered once with the dialect the table is +// for. Shared by the Postgres and SQLite `table`. + +import { compile as compileFragment } from "../sql/sql-fragment" +import { withDialect, type Dialect } from "../ch/dialect" +import type { Condition, Expr } from "../ch/expr" +import { createColumnAccessor, type ColumnAccessor } from "../ch/query" +import { columnSqlName, type ColumnDefs } from "../ch/types" +import type { DdlExpr, DdlKey } from "./define" + +/** A partial index's predicate: SQL, or a condition built with the DSL. */ +export type DdlPredicate = string | (($: ColumnAccessor) => Condition | Expr | string) + +export interface DdlRenderer { + /** SQL as written, or a DSL value rendered with the dialect. */ + readonly value: (value: Expr | Condition | string) => string + readonly expr: (expr: DdlExpr | DdlPredicate, columns: Cols) => string + /** Key parts: a column name is written quoted, an expression as it renders. */ + readonly keyParts: (key: DdlKey, columns: Cols) => ReadonlyArray +} + +export const ddlRenderer = (dialect: Dialect): DdlRenderer => { + const value = (v: Expr | Condition | string): string => + typeof v === "string" ? v : withDialect(dialect, () => compileFragment(v.toFragment())) + return { + value, + expr: (expr, columns) => (typeof expr === "string" ? expr : value(expr(createColumnAccessor(columns)))), + keyParts: (key, columns) => + typeof key === "function" + ? key(createColumnAccessor(columns)).map(value) + : key.map((column) => dialect.quoteIdent(columnSqlName(columns, column))), + } +} diff --git a/src/schema/entities.ts b/src/schema/entities.ts index 66788cd..a625276 100644 --- a/src/schema/entities.ts +++ b/src/schema/entities.ts @@ -9,6 +9,7 @@ import { Schema } from "effect" import { PgSchemaEntity } from "./pg-entities" +import type { SqliteSchemaEntity } from "./sqlite-entities" /** `DEFAULT`, `MATERIALIZED`, or `ALIAS`, with its SQL expression. */ export const ColumnDefault = Schema.Struct({ @@ -112,8 +113,11 @@ export type SchemaDialect = Snapshot["dialect"] /** An entity of either dialect. Everything dialect-neutral (keys, ordering, hashing, the branch graph) takes this. */ export type AnySchemaEntity = SchemaEntity | PgSchemaEntity +/** What `entityKey` and `sortEntities` take: a snapshot dialect's entity, or a SQLite one (no snapshot yet). */ +type KeyedEntity = AnySchemaEntity | SqliteSchemaEntity + /** A stable key per entity, unique within a snapshot: `:`, or `:
.` for a table's parts. */ -export const entityKey = (entity: AnySchemaEntity): string => +export const entityKey = (entity: KeyedEntity): string => "table" in entity ? `${entity.kind}:${entity.table}.${entity.name}` : entity.kind === "enum" && entity.schema !== null @@ -131,7 +135,7 @@ const kindOrder: Record = { } /** Enums, tables, then columns in declaration order, then indexes, unique constraints, foreign keys, views. Deterministic. */ -export const sortEntities = (entities: ReadonlyArray): ReadonlyArray => +export const sortEntities = (entities: ReadonlyArray): ReadonlyArray => [...entities].sort((a, b) => { const byKind = kindOrder[a.kind] - kindOrder[b.kind] if (byKind !== 0) return byKind diff --git a/src/schema/pg-define.ts b/src/schema/pg-define.ts index 956d461..8030276 100644 --- a/src/schema/pg-define.ts +++ b/src/schema/pg-define.ts @@ -8,17 +8,15 @@ // become DDL records a problem, and `pgEntitiesOf` fails with it. import { Schema } from "effect" -import { compile as compileFragment } from "../sql/sql-fragment" import { withDialect } from "../ch/dialect" -import type { Condition, Expr } from "../ch/expr" import { encodeColumnLiteral } from "../ch/literal" -import { createColumnAccessor, type ColumnAccessor } from "../ch/query" import type { Table } from "../ch/table" import { columnSqlName, type CHType, type ColumnDefs, type InferTS } from "../ch/types" import type { PgType } from "../pg/types" import { postgresDialect } from "../pg/dialect" +import { ddlRenderer, type DdlPredicate } from "./ddl-expr" import { externalTable, type DdlExpr, type DdlKey } from "./define" -import { withRenderProblems, type DefinitionProblem, type ProblemSink } from "./problems" +import { checkDistinctNames, checkNoRepeats, withRenderProblems, type DefinitionProblem, type ProblemSink } from "./problems" import { canonicalPgType, PG_MAX_IDENTIFIER, @@ -33,22 +31,11 @@ import { type PgUniqueEntity, } from "./pg-entities" -const quoteIdent = (name: string): string => `"${name.replace(/"/g, '""')}"` +const ddl = ddlRenderer(postgresDialect) +const renderExpr = ddl.expr +const renderKeyParts = ddl.keyParts -const renderValue = (value: Expr | Condition | string): string => - typeof value === "string" ? value : withDialect(postgresDialect, () => compileFragment(value.toFragment())) - -/** A partial index's predicate: SQL, or a condition built with the DSL. */ -export type DdlPredicate = string | (($: ColumnAccessor) => Condition | Expr | string) - -const renderExpr = (expr: DdlExpr | DdlPredicate, columns: Cols): string => - typeof expr === "string" ? expr : renderValue(expr(createColumnAccessor(columns))) - -/** Key parts: a column name is written quoted, an expression as it renders. */ -const renderKeyParts = (key: DdlKey, columns: Cols): ReadonlyArray => - typeof key === "function" - ? key(createColumnAccessor(columns)).map(renderValue) - : key.map((column) => quoteIdent(columnSqlName(columns, column))) +export type { DdlPredicate } // Columns @@ -482,8 +469,12 @@ function buildTable [column, sqlName(column)] as const), { ignoreCase: false }) + const pk = definition.primaryKey const pkColumns = pk === undefined ? [] : "columns" in pk ? pk.columns : pk + checkNoRepeats(problems, `${name} primary key`, pkColumns) const pkName = pk !== undefined && "columns" in pk && pk.name !== undefined ? pk.name : `${name}_pkey` if (pk !== undefined) { checkIdentifier(problems, `${name} primary key`, pkName) @@ -502,6 +493,7 @@ function buildTable { checkIdentifier(problems, `${name} index`, spec.name) if (Array.isArray(spec.on)) { + checkNoRepeats(problems, `${name} index ${spec.name}`, spec.on) for (const column of spec.on) { if (!has(column)) problems.push({ object: `${name} index ${spec.name}`, message: `${column} is not a column` }) } diff --git a/src/schema/pg-schema.test.ts b/src/schema/pg-schema.test.ts index d1eb7e3..44e8d14 100644 --- a/src/schema/pg-schema.test.ts +++ b/src/schema/pg-schema.test.ts @@ -135,6 +135,10 @@ describe("PG.table", () => { it("rejects definitions Postgres would not take as written", () => { expect(problemsOf(PG.table("t", { columns: { a: PG.nullable(PG.text) }, primaryKey: ["a"] }))).toMatch(/cannot be nullable/) expect(problemsOf(PG.table("t".repeat(64), { columns: { a: PG.text } }))).toMatch(/longer than 63/) + // Each renders, then Postgres refuses it: "column specified more than once". + expect(problemsOf(PG.table("t", { columns: { a: PG.column(PG.text, { name: "b" }), b: PG.text } }))).toMatch(/b has the same SQL name as a/) + expect(problemsOf(PG.table("t", { columns: { a: PG.text }, primaryKey: ["a", "a"] }))).toMatch(/a is listed more than once/) + expect(problemsOf(PG.table("t", { columns: { a: PG.text }, indexes: [PG.index("t_a", ["a", "a"])] }))).toMatch(/a is listed more than once/) expect( problemsOf( PG.table("t", { columns: { a: PG.text }, foreignKeys: [PG.foreignKey({ columns: ["a"], references: "u", foreignColumns: ["x", "y"] })] }), diff --git a/src/schema/problems.ts b/src/schema/problems.ts index 6286ee9..ca31993 100644 --- a/src/schema/problems.ts +++ b/src/schema/problems.ts @@ -31,6 +31,32 @@ export const checkIdentifier = (problems: ProblemSink, object: string, name: str } /** Build a definition, adding any failure its rendering recorded to its `problems`. */ +/** + * Two columns of one table with the same SQL name. `ignoreCase` where the + * database compares names without case (SQLite), so `Id` and `id` collide. + */ +export const checkDistinctNames = ( + problems: ProblemSink, + table: string, + columns: ReadonlyArray, + options: { readonly ignoreCase: boolean }, +): void => { + const seen = new Map() + for (const [key, sqlName] of columns) { + const folded = options.ignoreCase ? sqlName.toLowerCase() : sqlName + const other = seen.get(folded) + if (other !== undefined) { + problems.push({ object: `${table}.${key}`, message: `${sqlName} has the same SQL name as ${other}${options.ignoreCase ? " (names compare without case)" : ""}` }) + } else seen.set(folded, key) + } +} + +/** A key (primary key, index) that lists a column more than once. */ +export const checkNoRepeats = (problems: ProblemSink, object: string, keys: ReadonlyArray): void => { + const repeats = keys.filter((key, i) => keys.indexOf(key) !== i) + if (repeats.length > 0) problems.push({ object, message: `${[...new Set(repeats)].join(", ")} is listed more than once` }) +} + export const withRenderProblems = }>(object: string, build: () => O): O => { const { value, failures } = collectFailures(build) return failures.length === 0 diff --git a/src/schema/sqlite-define.ts b/src/schema/sqlite-define.ts new file mode 100644 index 0000000..0953760 --- /dev/null +++ b/src/schema/sqlite-define.ts @@ -0,0 +1,376 @@ +// SQLite schema definitions: `table` from `/sqlite` with its column and index +// helpers. +// +// The SQLite counterpart of the Postgres `table`: the value IS a `Table`, so +// every query API accepts it, and its DDL rides beside it on `ddl` as SQLite +// entities. A definition that cannot become DDL records a problem, and +// `sqliteEntitiesOf` fails with it. + +import { Effect } from "effect" +import type { Table } from "../ch/table" +import { withDialect } from "../ch/dialect" +import { encodeColumnLiteral } from "../ch/literal" +import { columnSqlName, type CHType, type ColumnDefs, type InferTS } from "../ch/types" +import { sqliteDialect } from "../sqlite/dialect" +import { timestamp } from "../sqlite/types" +import { createColumnAccessor } from "../ch/query" +import { ddlRenderer } from "./ddl-expr" +import { externalTable, type DdlExpr, type DdlKey } from "./define" +import { sortEntities } from "./entities" +import { checkDistinctNames, checkIdentifier, checkNoRepeats, definitionError, withRenderProblems, type DefinitionProblem, type ProblemSink, type SchemaDefinitionError } from "./problems" +import { + canonicalSqliteType, + type SqliteColumnEntity, + type SqliteIndexEntity, + type SqliteSchemaEntity, + type SqliteTableEntity, + type SqliteUniqueEntity, +} from "./sqlite-entities" + +const ddl = ddlRenderer(sqliteDialect) + +// Columns + +export interface ColumnOptions> { + /** + * The column's name in the database, when it differs from its key in + * `columns`: `orgId: SQLite.column(SQLite.text, { name: "org_id" })` is read + * and written as `$.orgId` and stored as `org_id`. + */ + readonly name?: string + /** A literal default, encoded through the column's own type. A boolean is written `true` / `false`. */ + readonly default?: InferTS + /** + * `DEFAULT `. A string is written as given, so put an expression in + * the parentheses SQLite requires: `"(unixepoch())"`. A DSL expression is + * parenthesized for you. + */ + readonly defaultExpr?: DdlExpr + /** + * A one-column `UNIQUE` constraint, written inline as drizzle-kit 1.0 writes + * `.unique()`. `true` names it `
__unique`, drizzle's name; a + * string names it. SQLite keeps no constraint name; the snapshot does. + */ + readonly unique?: true | string +} + +/** `Given` records which options were set, so the table knows which columns an insert may leave out. */ +export interface ColumnSpec< + T extends CHType, + Given extends keyof ColumnOptions = keyof ColumnOptions, +> { + readonly _tag: "SqliteColumnSpec" + readonly type: T + readonly options: ColumnOptions + readonly _given?: Given +} + +/** A column with options. A bare column type works too where none is needed. */ +export const column = , Given extends keyof ColumnOptions = never>( + type: T, + // A default must be a value: `default: undefined` would make the column optional on insert + // while writing no DEFAULT, so the insert would fail on NOT NULL. + options: ColumnOptions & { readonly [K in Given]: K extends "default" | "defaultExpr" ? {} : unknown } = {} as never, +): ColumnSpec => ({ _tag: "SqliteColumnSpec", type, options }) + +export type ColumnInput = CHType | ColumnSpec, any> + +/** The query-side column types of a `columns` record. */ +export type ColumnsOf> = { + readonly [K in keyof I]: I[K] extends ColumnSpec ? T : I[K] extends CHType ? I[K] : never +} + +/** Columns declared with `default` or `defaultExpr`: an insert may leave them out. */ +export type DefaultedColumnsOf> = { + [K in keyof I]: I[K] extends ColumnSpec + ? [Exclude] extends [never] + ? never + : K + : never +}[keyof I] & + string + +const isColumnSpec = (input: unknown): input is ColumnSpec, any> => + typeof input === "object" && input !== null && "_tag" in input && input._tag === "SqliteColumnSpec" + +const columnTypeOfInput = (input: ColumnInput): CHType => { + if (!isColumnSpec(input)) return input + return input.options.name === undefined ? input.type : { ...input.type, sqlName: input.options.name } +} + +// Indexes + +export interface IndexSpec { + readonly _tag: "SqliteIndexSpec" + readonly name: string + readonly unique: boolean + readonly on: DdlKey +} + +/** An index on columns or expressions: `index("t_org", ["orgId"])`, or a callback building expressions. */ +export const index = (name: string, on: DdlKey): IndexSpec => ({ + _tag: "SqliteIndexSpec", + name, + unique: false, + on, +}) + +/** A unique index. An `onConflict` target can name its columns. */ +export const uniqueIndex = (name: string, on: DdlKey): IndexSpec => ({ + _tag: "SqliteIndexSpec", + name, + unique: true, + on, +}) + +/** A `SQLite.timestamp` column, nullable or not. */ +const isTimestamp = (type: CHType): boolean => + type.schema === timestamp.schema || (type._tag === "Nullable" && (type as { element?: CHType }).element?.schema === timestamp.schema) + +/** + * SQLite compares timestamp text by bytes, so a timestamp column's default + * must write the column's own form (`timestampNow`). CURRENT_TIMESTAMP, + * datetime(), unixepoch() and a strftime with another format write another + * form, which misorders against the column's values. + */ +const checkTimestampDefault = (problems: ProblemSink, table: string, name: string, spec: ColumnSpec>, sql: string): void => { + if (!isTimestamp(spec.type)) return + const otherForm = + /\bcurrent_(timestamp|date|time)\b|\b(datetime|date|time|julianday|unixepoch)\s*\(/i.test(sql) || + /\bstrftime\s*\(\s*'(?!%Y-%m-%d %H:%M:%f')/i.test(sql) + if (otherForm) { + problems.push({ + object: `${table}.${name}`, + message: `defaultExpr ${sql} writes a different text form than a timestamp column compares with; use SQLite.timestampNow`, + }) + } +} + +const columnDefault = ( + problems: ProblemSink, + table: string, + name: string, + spec: ColumnSpec>, + columns: ColumnDefs, +): string | null => { + const { options } = spec + if (options.default !== undefined && options.defaultExpr !== undefined) { + problems.push({ object: `${table}.${name}`, message: "a column takes one of default, defaultExpr" }) + } + for (const key of ["default", "defaultExpr"] as const) { + if (Object.hasOwn(options, key) && options[key] === undefined) { + problems.push({ object: `${table}.${name}`, message: `${key} is undefined; leave it out, or the column is optional on insert with no DEFAULT` }) + } + } + if (options.defaultExpr !== undefined) { + if (typeof options.defaultExpr === "string") { + checkTimestampDefault(problems, table, name, spec, options.defaultExpr) + return options.defaultExpr + } + // SQLite refuses a DEFAULT that reads a column ("default value of column is not constant"). + const read: Array = [] + const accessor = createColumnAccessor(columns) + const tracked = new Proxy(accessor, { + get: (target, key, receiver) => { + if (typeof key === "string" && Object.hasOwn(columns, key)) read.push(key) + return Reflect.get(target, key, receiver) + }, + }) + const sql = ddl.value(options.defaultExpr(tracked as typeof accessor)) + if (read.length > 0) { + problems.push({ object: `${table}.${name}`, message: `defaultExpr reads ${[...new Set(read)].join(", ")}; SQLite requires a constant default` }) + } + checkTimestampDefault(problems, table, name, spec, sql) + return `(${sql})` + } + if (options.default === undefined || options.default === null) return null + // SQLite reads TRUE / FALSE as 1 / 0; drizzle-kit writes them as keywords. + if (typeof options.default === "boolean") return options.default ? "true" : "false" + return withDialect(sqliteDialect, () => encodeColumnLiteral(spec.type, options.default, name)) +} + +// Tables + +export interface TableDefinition> { + readonly columns: Columns + /** One column, written inline as drizzle-kit does, or several, as a table constraint. */ + readonly primaryKey?: ReadonlyArray + readonly indexes?: ReadonlyArray>> + /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ + readonly tenantColumn?: keyof Columns & string +} + +/** A table this schema does not own: a view, or one another tool migrates. No DDL. */ +export interface ExternalTableDefinition> { + readonly external: true + readonly columns: Columns + readonly tenantColumn?: keyof Columns & string +} + +/** The DDL a SQLite table carries, as entities. */ +export interface TableDdl { + readonly dialect: "sqlite" + readonly table: SqliteTableEntity + readonly columns: ReadonlyArray + readonly indexes: ReadonlyArray + readonly uniques: ReadonlyArray +} + +export interface SqliteSchemaTable + extends Table { + readonly ddl: TableDdl + /** What is wrong with the definition; `sqliteEntitiesOf` fails when any table has one. */ + readonly problems: ReadonlyArray +} + +/** A table or index name SQLite reserves for itself. */ +const checkName = (problems: ProblemSink, object: string, name: string): void => { + checkIdentifier(problems, object, name) + if (name.toLowerCase().startsWith("sqlite_")) { + problems.push({ object, message: `${JSON.stringify(name)} starts with sqlite_, which SQLite reserves` }) + } +} + +/** Is `definition` a `{ columns, ... }` definition, rather than a record of columns? */ +export const isTableDefinition = (definition: object): definition is { readonly columns: Record } => { + if (!("columns" in definition)) return false + const columns = definition.columns + return typeof columns === "object" && columns !== null && !("sql" in columns && "schema" in columns) +} + +export const externalSqliteTable = ( + name: Name, + definition: ExternalTableDefinition>, +): Table => externalTable(name, definition, isColumnSpec, [], (spec) => columnTypeOfInput(spec as ColumnInput)) + +/** + * A SQLite table with its DDL. The value IS a query `Table`. + * + * A column is `NOT NULL` unless its type is `SQLite.nullable(...)`. + */ +export const sqliteTable = >( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> => withRenderProblems(name, () => buildTable(name, definition)) + +function buildTable>( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> { + const problems: ProblemSink = [] + checkName(problems, name, name) + const inputs = Object.entries(definition.columns) + if (inputs.length === 0) problems.push({ object: name, message: "a table needs columns" }) + const types = Object.fromEntries(inputs.map(([column, input]) => [column, columnTypeOfInput(input)])) as ColumnsOf + const sqlName = (column: string) => columnSqlName(types, column) + const has = (column: string) => Object.hasOwn(types, column) + + const columnEntities = inputs.map(([column, input], position): SqliteColumnEntity => { + checkIdentifier(problems, `${name}.${column}`, sqlName(column)) + const spec: ColumnSpec> = isColumnSpec(input) + ? input + : { _tag: "SqliteColumnSpec", type: input as CHType, options: {} } + return { + kind: "column", + table: name, + name: sqlName(column), + position, + type: canonicalSqliteType(spec.type.sql), + notNull: spec.type._tag !== "Nullable", + default: columnDefault(problems, name, column, spec, types), + } + }) + + // SQLite compares column names without case, so `Id` and `id` are one column. + checkDistinctNames(problems, name, inputs.map(([column]) => [column, sqlName(column)] as const), { ignoreCase: true }) + + const pkColumns = definition.primaryKey ?? [] + checkNoRepeats(problems, `${name} primary key`, pkColumns) + if (definition.primaryKey !== undefined) { + if (pkColumns.length === 0) problems.push({ object: name, message: "a primary key needs columns" }) + for (const column of pkColumns) { + if (!has(column)) problems.push({ object: `${name} primary key`, message: `${column} is not a column` }) + else if (columnEntities.find((c) => c.name === sqlName(column))?.notNull === false) { + problems.push({ + object: `${name}.${column}`, + message: "a primary key column cannot be nullable; SQLite would accept NULL keys", + }) + } + } + } + + const indexEntities: ReadonlyArray = (definition.indexes ?? []).map((spec): SqliteIndexEntity => { + checkName(problems, `${name} index`, spec.name) + if (Array.isArray(spec.on)) { + checkNoRepeats(problems, `${name} index ${spec.name}`, spec.on) + for (const column of spec.on) { + if (!has(column)) problems.push({ object: `${name} index ${spec.name}`, message: `${column} is not a column` }) + } + } + const columns = ddl.keyParts(spec.on, types) + if (columns.length === 0) problems.push({ object: `${name} index ${spec.name}`, message: "an index needs columns" }) + return { kind: "index", table: name, name: spec.name, unique: spec.unique, columns } + }) + + const uniqueEntities: Array = [] + for (const [column, input] of inputs) { + const option = isColumnSpec(input) ? input.options.unique : undefined + if (option === undefined) continue + const uniqueName = option === true ? `${name}_${sqlName(column)}_unique` : option + checkIdentifier(problems, `${name}.${column} unique`, uniqueName) + uniqueEntities.push({ kind: "unique", table: name, name: uniqueName, columns: [sqlName(column)] }) + } + + const tableEntity: SqliteTableEntity = { + kind: "table", + name, + primaryKey: definition.primaryKey === undefined ? null : { columns: pkColumns.filter(has).map(sqlName) }, + } + const defaults = inputs.filter((_, position) => columnEntities[position]!.default !== null).map(([column]) => column) + return { + _tag: "Table", + name, + columns: types, + ...(definition.tenantColumn !== undefined ? { tenantColumn: definition.tenantColumn } : undefined), + ...(defaults.length > 0 ? { defaults: defaults as unknown as Array> } : undefined), + ddl: { dialect: "sqlite", table: tableEntity, columns: columnEntities, indexes: indexEntities, uniques: uniqueEntities }, + problems, + } +} + +/** + * The entities of a set of SQLite tables, validated as one schema: table and + * index names do not collide (SQLite keeps them in one namespace). Fails with + * every problem found. + */ +export const sqliteEntitiesOf = ( + tables: ReadonlyArray>, +): Effect.Effect, SchemaDefinitionError> => { + const problems: ProblemSink = [] + const entities: Array = [] + const names = new Map() + const claim = (name: string, what: string) => { + const other = names.get(name.toLowerCase()) + if (other !== undefined) { + problems.push({ object: name, message: `names both ${other} and ${what}; SQLite keeps them in one namespace` }) + } + names.set(name.toLowerCase(), what) + } + for (const table of tables) { + if (!("ddl" in table) || (table.ddl as { dialect?: string }).dialect !== "sqlite") { + problems.push({ object: table.name, message: "is not a SQLite table with DDL; declare it with SQLite.table(name, { columns })" }) + continue + } + problems.push(...table.problems) + const { ddl } = table + claim(ddl.table.name, `table ${ddl.table.name}`) + for (const index of ddl.indexes) claim(index.name, `an index on ${ddl.table.name}`) + const constraints = new Set() + for (const unique of ddl.uniques) { + if (constraints.has(unique.name)) problems.push({ object: `${ddl.table.name}.${unique.name}`, message: "defined twice" }) + constraints.add(unique.name) + } + entities.push(ddl.table, ...ddl.columns, ...ddl.indexes, ...ddl.uniques) + } + return problems.length > 0 ? Effect.fail(definitionError(problems)) : Effect.succeed(sortEntities(entities)) +} diff --git a/src/schema/sqlite-entities.ts b/src/schema/sqlite-entities.ts new file mode 100644 index 0000000..9d2493f --- /dev/null +++ b/src/schema/sqlite-entities.ts @@ -0,0 +1,63 @@ +// SQLite schema entities: the normalized, serializable form of a SQLite +// schema, as `entities.ts` is for ClickHouse and `pg-entities.ts` for Postgres. +// +// Types are stored as the declared type, lowercase (`text`, `integer`), which +// is what `PRAGMA table_info` reports back and what drizzle-kit writes. +// Expressions (defaults, index key parts) are SQL as written. + +import { Schema } from "effect" + +export const SqliteTableEntity = Schema.Struct({ + kind: Schema.Literal("table"), + name: Schema.String, + /** + * The primary key columns, or `null` for a table without one. SQLite keeps + * no name for the constraint, so none is stored. + */ + primaryKey: Schema.NullOr(Schema.Struct({ columns: Schema.Array(Schema.String) })), +}) +export type SqliteTableEntity = typeof SqliteTableEntity.Type + +export const SqliteColumnEntity = Schema.Struct({ + kind: Schema.Literal("column"), + table: Schema.String, + name: Schema.String, + /** Declaration order, the order `CREATE TABLE` writes the columns in. */ + position: Schema.Number, + /** The declared type, lowercase: `text`, `integer`, `real`, `blob`, `numeric`. */ + type: Schema.String, + notNull: Schema.Boolean, + /** The `DEFAULT` value as SQL: a literal, or an expression in parentheses. `null` for none. */ + default: Schema.NullOr(Schema.String), +}) +export type SqliteColumnEntity = typeof SqliteColumnEntity.Type + +/** An index, `CREATE [UNIQUE] INDEX`. */ +export const SqliteIndexEntity = Schema.Struct({ + kind: Schema.Literal("index"), + table: Schema.String, + name: Schema.String, + unique: Schema.Boolean, + /** Key parts as SQL: a quoted column (`"org_id"`) or an expression. */ + columns: Schema.Array(Schema.String), +}) +export type SqliteIndexEntity = typeof SqliteIndexEntity.Type + +/** + * A `UNIQUE` constraint, written in `CREATE TABLE` as drizzle-kit 1.0 writes + * `.unique()`. SQLite backs it with an automatic index and keeps no name for + * it, so `name` lives only here. Changing one rebuilds the table. + */ +export const SqliteUniqueEntity = Schema.Struct({ + kind: Schema.Literal("unique"), + table: Schema.String, + name: Schema.String, + columns: Schema.Array(Schema.String), +}) +export type SqliteUniqueEntity = typeof SqliteUniqueEntity.Type + +export const SqliteSchemaEntity = Schema.Union([SqliteTableEntity, SqliteColumnEntity, SqliteIndexEntity, SqliteUniqueEntity]) +export type SqliteSchemaEntity = typeof SqliteSchemaEntity.Type + +/** The declared type of a column, as `PRAGMA table_info` reports it back. */ +export const canonicalSqliteType = (type: string): string => type.trim().replace(/\s+/g, " ").toLowerCase() diff --git a/src/schema/sqlite-render.ts b/src/schema/sqlite-render.ts new file mode 100644 index 0000000..20f871c --- /dev/null +++ b/src/schema/sqlite-render.ts @@ -0,0 +1,59 @@ +// SQLite DDL rendering from entities: `CREATE TABLE` and `CREATE [UNIQUE] INDEX`. + +import { sqliteDialect } from "../sqlite/dialect" +import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" + +/** An identifier, always double-quoted, so mixed case and keywords survive. */ +export const sqliteIdent = (name: string): string => sqliteDialect.quoteIdent(name) + +const list = (names: ReadonlyArray): string => names.map(sqliteIdent).join(", ") + +/** + * One column's definition. `primaryKey` writes the single-column key inline and + * `unique` a one-column `UNIQUE` constraint, as drizzle-kit 1.0 does. + */ +export const renderSqliteColumnDefinition = (column: SqliteColumnEntity, primaryKey = false, unique = false): string => { + const parts = [sqliteIdent(column.name), column.type] + if (primaryKey) parts.push("PRIMARY KEY") + if (unique) parts.push("UNIQUE") + if (column.notNull) parts.push("NOT NULL") + if (column.default !== null) parts.push(`DEFAULT ${column.default}`) + return parts.join(" ") +} + +export const renderSqliteCreateTable = ( + table: SqliteTableEntity, + columns: ReadonlyArray, + uniques: ReadonlyArray = [], +): string => { + const key = table.primaryKey?.columns ?? [] + const inline = key.length === 1 ? key[0] : undefined + const inlineUnique = new Set(uniques.filter((u) => u.columns.length === 1).map((u) => u.columns[0])) + const body = [...columns] + .sort((a, b) => a.position - b.position) + .map((column) => renderSqliteColumnDefinition(column, column.name === inline, inlineUnique.has(column.name))) + if (key.length > 1) body.push(`PRIMARY KEY (${list(key)})`) + for (const unique of uniques) if (unique.columns.length > 1) body.push(`UNIQUE (${list(unique.columns)})`) + return `CREATE TABLE IF NOT EXISTS ${sqliteIdent(table.name)} (\n\t${body.join(",\n\t")}\n)` +} + +export const renderSqliteCreateIndex = (index: SqliteIndexEntity): string => + `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${sqliteIdent(index.name)} ON ${sqliteIdent(index.table)} (${index.columns.join(", ")})` + +/** Every CREATE statement for a schema: tables, then indexes. */ +export const renderSqliteSchema = (entities: ReadonlyArray): ReadonlyArray => { + const columns = entities.filter((e): e is SqliteColumnEntity => e.kind === "column") + const uniques = entities.filter((e): e is SqliteUniqueEntity => e.kind === "unique") + return [ + ...entities + .filter((e): e is SqliteTableEntity => e.kind === "table") + .map((table) => + renderSqliteCreateTable( + table, + columns.filter((c) => c.table === table.name), + uniques.filter((u) => u.table === table.name), + ), + ), + ...entities.filter((e): e is SqliteIndexEntity => e.kind === "index").map(renderSqliteCreateIndex), + ] +} diff --git a/src/sqlite.ts b/src/sqlite.ts index d8c0df2..de5b1f7 100644 --- a/src/sqlite.ts +++ b/src/sqlite.ts @@ -1,34 +1,60 @@ // @maple-dev/effect-orm/sqlite // -// SQLite and Cloudflare D1 queries, codecs, functions and execution through an -// Effect SqlClient. `table` is a query schema only: this entry writes no DDL -// and generates no migrations yet. See docs/sqlite.md. +// SQLite and Cloudflare D1 from one import: column types, tables with their +// DDL, SQLite functions, the query builder and a `compile` that defaults to +// the SQLite dialect. Migrations stay with the caller for now. See +// docs/sqlite.md. import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" import { compiled, compiledUnsafe } from "./ch/failure" import { table as queryTable, type Table, type TableOptions } from "./ch/table" import type { ColumnDefs } from "./ch/types" +import { + externalSqliteTable, + isTableDefinition, + sqliteTable, + type ColumnInput, + type ColumnsOf, + type DefaultedColumnsOf, + type ExternalTableDefinition, + type SqliteSchemaTable, + type TableDefinition, +} from "./schema/sqlite-define" import { sqliteDialect } from "./sqlite/dialect" export * from "./core" export * from "./sqlite/types" export * from "./sqlite/functions" export type { TableOptions } - -/** A table another tool owns, as on `/clickhouse` and `/postgres`. */ -export interface ExternalTableDefinition { - readonly external: true - readonly columns: Columns - readonly tenantColumn?: keyof Columns & string -} +export { + column, + index, + uniqueIndex, + type ColumnInput, + type ColumnOptions, + type ColumnSpec, + type ColumnsOf, + type DefaultedColumnsOf, + type ExternalTableDefinition, + type IndexSpec, + type SqliteSchemaTable, + type TableDdl, + type TableDefinition, +} from "./schema/sqlite-define" +export { SchemaDefinitionError, type DefinitionProblem, type DdlExpr, type DdlKey } from "./schema/define" /** - * A SQLite query schema: `table(name, columns, options?)`, or - * `table(name, { external: true, columns, tenantColumn? })`. Neither carries DDL. + * A SQLite table. With `{ columns, primaryKey, indexes }` it carries its DDL + * (`renderSqliteSchema` from `/schema` writes it). With `{ external: true, columns }`, + * or a bare record of columns and query options, it is a query schema only. */ -export function table( +export function table>( name: Name, definition: ExternalTableDefinition, -): Table +): Table, DefaultedColumnsOf, never> +export function table>( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> export function table< const Name extends string, const Columns extends ColumnDefs, @@ -36,10 +62,9 @@ export function table< const Computed extends keyof Columns & string = never, >(name: Name, columns: Columns, options?: TableOptions): Table export function table(name: string, definition: any, options?: TableOptions): Table { - if (options === undefined && definition.external === true && typeof definition.columns === "object") { - return queryTable(name, definition.columns, definition.tenantColumn === undefined ? undefined : { tenantColumn: definition.tenantColumn }) - } - return queryTable(name, definition, options) + if (options !== undefined || !isTableDefinition(definition)) return queryTable(name, definition, options) + if ("external" in definition && definition.external === true) return externalSqliteTable(name, definition as ExternalTableDefinition) + return sqliteTable(name, definition as TableDefinition) } export { sqliteDialect, d1Dialect } from "./sqlite/dialect" diff --git a/src/sqlite/ddl.test.ts b/src/sqlite/ddl.test.ts new file mode 100644 index 0000000..47e8593 --- /dev/null +++ b/src/sqlite/ddl.test.ts @@ -0,0 +1,257 @@ +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { DateTime, Effect, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { expect, it } from "vitest" +import * as Db from "../database" +import { canonicalJson, renderSqliteSchema, SqliteSchemaEntity, sqliteEntitiesOf } from "../schema" +import * as SQLite from "../sqlite" + +// drizzle-kit 0.31.11 (drizzle-orm 0.45.4) `generate` output for the same table. +const drizzleShortLink = `CREATE TABLE \`short_link\` ( + \`id\` text PRIMARY KEY NOT NULL, + \`code\` text NOT NULL, + \`target_url\` text NOT NULL, + \`target_hash\` text NOT NULL, + \`org_id\` text, + \`created_by\` text, + \`subject_type\` text NOT NULL, + \`subject_id\` text NOT NULL, + \`purpose\` text NOT NULL, + \`prospect_id\` text, + \`click_count\` integer NOT NULL, + \`first_accessed_at\` text, + \`last_accessed_at\` text, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX \`short_link_code\` ON \`short_link\` (\`code\`);--> statement-breakpoint +CREATE INDEX \`short_link_org\` ON \`short_link\` (\`org_id\`);` + +const named = [0]>(type: T, name: string) => SQLite.column(type, { name }) +const optionalText = (name: string) => named(SQLite.nullable(SQLite.text), name) + +const ShortLink = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.text, + targetUrl: named(SQLite.text, "target_url"), + targetHash: named(SQLite.text, "target_hash"), + orgId: optionalText("org_id"), + createdBy: optionalText("created_by"), + subjectType: named(SQLite.text, "subject_type"), + subjectId: named(SQLite.text, "subject_id"), + purpose: SQLite.text, + prospectId: optionalText("prospect_id"), + clickCount: named(SQLite.integer, "click_count"), + firstAccessedAt: optionalText("first_accessed_at"), + lastAccessedAt: optionalText("last_accessed_at"), + createdAt: named(SQLite.text, "created_at"), + updatedAt: named(SQLite.text, "updated_at"), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) + +const drizzleStatements = (sql: string) => sql.split("--> statement-breakpoint").map((s) => s.trim()).filter((s) => s.length > 0) + +/** What SQLite reports about every table and index after running `statements` on an empty database. */ +const shapeAfter = (statements: ReadonlyArray) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + for (const sql of statements) yield* db.execute({ sql }) + const tables = yield* db.query({ sql: "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name" }) + const shape: Record = {} + for (const { name } of tables as ReadonlyArray<{ name: string }>) { + const columns = yield* db.query({ sql: `SELECT * FROM pragma_table_info('${name}')` }) + const list = yield* db.query({ sql: `SELECT name, "unique", origin, partial FROM pragma_index_list('${name}') ORDER BY name` }) + const indexes = [] + for (const index of list as ReadonlyArray<{ name: string }>) { + const keys = yield* db.query({ sql: `SELECT seqno, cid, name, "desc", coll, "key" FROM pragma_index_xinfo('${index.name}')` }) + indexes.push({ ...index, keys }) + } + shape[name] = { columns, indexes } + } + return shape + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + +const ddlOf = (tables: Parameters[0]) => Effect.runPromise(Effect.map(sqliteEntitiesOf(tables), renderSqliteSchema)) + +it("creates short_link exactly as drizzle-kit's SQL does, by PRAGMA table_info and index_list", async () => { + const ours = await shapeAfter(await ddlOf([ShortLink])) + expect(ours).toEqual(await shapeAfter(drizzleStatements(drizzleShortLink))) + expect(Object.keys(ours)).toEqual(["short_link"]) +}) + +// drizzle-kit 1.0.0-rc.5 for: composite primaryKey, .default(...) literals, sql`(CURRENT_TIMESTAMP)`, +// text("email").unique(), real, blob, numeric, a two-column index. drizzle-kit 1.0 writes +// .unique() inline, as a UNIQUE constraint; 0.31 wrote a separate unique index. +const drizzleMembership = `CREATE TABLE \`membership\` ( + \`org_id\` text NOT NULL, + \`user_id\` text NOT NULL, + \`role\` text DEFAULT 'member' NOT NULL, + \`weight\` real DEFAULT 1.5 NOT NULL, + \`active\` integer DEFAULT true NOT NULL, + \`avatar\` blob, + \`amount\` numeric, + \`email\` text UNIQUE, + \`created_at\` text DEFAULT (CURRENT_TIMESTAMP) NOT NULL, + CONSTRAINT \`membership_pk\` PRIMARY KEY(\`org_id\`, \`user_id\`) +); +--> statement-breakpoint +CREATE INDEX \`membership_user_role\` ON \`membership\` (\`user_id\`,\`role\`);` + +// drizzle-kit 1.0.0-rc.5 for short_link. It drops NOT NULL from a primary key column even when +// the column says .notNull(), so SQLite accepts a NULL id. We keep NOT NULL. +const drizzleV1ShortLinkTable = drizzleShortLink + .replace("`id` text PRIMARY KEY NOT NULL", "`id` text PRIMARY KEY") + .split("--> statement-breakpoint")[0]! + +const Membership = SQLite.table("membership", { + columns: { + orgId: named(SQLite.text, "org_id"), + userId: named(SQLite.text, "user_id"), + role: SQLite.column(SQLite.text, { default: "member" }), + weight: SQLite.column(SQLite.real, { default: 1.5 }), + active: SQLite.column(SQLite.boolean, { default: true }), + avatar: SQLite.nullable(SQLite.blob), + amount: SQLite.nullable(SQLite.custom("numeric", Schema.Finite)), + email: SQLite.column(SQLite.nullable(SQLite.text), { unique: true }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["orgId", "userId"], + indexes: [SQLite.index("membership_user_role", ["userId", "role"])], + tenantColumn: "orgId", +}) + +it("matches drizzle-kit 1.0 for short_link, except that the primary key column stays NOT NULL", async () => { + const ours = (await shapeAfter(await ddlOf([ShortLink]))) as { short_link: { columns: Array<{ name: string; notnull: number }> } } + const theirs = (await shapeAfter([drizzleV1ShortLinkTable, ...drizzleStatements(drizzleShortLink).slice(1)])) as typeof ours + expect(ours.short_link.columns.find((c) => c.name === "id")?.notnull).toBe(1) + expect(theirs.short_link.columns.find((c) => c.name === "id")?.notnull).toBe(0) + const withoutKeyNull = (shape: typeof ours) => ({ ...shape, short_link: { ...shape.short_link, columns: shape.short_link.columns.map((c) => (c.name === "id" ? { ...c, notnull: 1 } : c)) } }) + expect(withoutKeyNull(ours)).toEqual(withoutKeyNull(theirs)) +}) + +it("creates composite keys, defaults, unique columns and every storage class as drizzle-kit 1.0 does", async () => { + expect(await shapeAfter(await ddlOf([Membership]))).toEqual(await shapeAfter(drizzleStatements(drizzleMembership))) +}) + +it("lets an insert leave out defaulted columns, and SQLite fills them", async () => { + const rows = await Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + for (const sql of yield* Effect.map(sqliteEntitiesOf([Membership]), renderSqliteSchema)) yield* db.execute({ sql }) + yield* db.run(SQLite.insertInto(Membership).values({ orgId: "o", userId: "u", avatar: null, amount: null, email: null })) + return yield* db.run(SQLite.from(Membership).select("role", "weight", "active")) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + expect(rows).toEqual([{ role: "member", weight: 1.5, active: true }]) +}) + +it("renders DSL defaults and expression index keys that SQLite accepts", async () => { + const Codes = SQLite.table("codes", { + columns: { + id: SQLite.integer, + code: SQLite.text, + hits: SQLite.column(SQLite.integer, { defaultExpr: () => SQLite.lit(1) }), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("codes_code_lower", ($) => [SQLite.lower($.code)])], + }) + const statements = await ddlOf([Codes]) + expect(statements).toEqual([ + 'CREATE TABLE IF NOT EXISTS "codes" (\n\t"id" integer PRIMARY KEY NOT NULL,\n\t"code" text NOT NULL,\n\t"hits" integer NOT NULL DEFAULT (1)\n)', + 'CREATE UNIQUE INDEX IF NOT EXISTS "codes_code_lower" ON "codes" (lower("code"))', + ]) + const shape = (await shapeAfter(statements)) as { codes: { indexes: Array<{ keys: Array<{ cid: number }> }> } } + expect(shape.codes.indexes[0]?.keys[0]?.cid).toBe(-2) +}) + +it("keeps the query-only forms free of DDL", async () => { + const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer, orgId: named(SQLite.text, "org_id") }, tenantColumn: "orgId" }) + const Legacy = SQLite.table("legacy", { id: SQLite.integer, note: SQLite.text }, { defaults: ["note"] }) + expect("ddl" in External || "ddl" in Legacy).toBe(false) + expect(SQLite.compileUnsafe(SQLite.from(External).select("orgId").where(($) => [$.orgId.eq("o")])).sql).toContain('"ext"."org_id" AS "orgId"') + const exit = await Effect.runPromiseExit(sqliteEntitiesOf([External as never])) + expect(exit._tag).toBe("Failure") +}) + +it("records every definition problem, and sqliteEntitiesOf fails with all of them", async () => { + const Bad = SQLite.table("sqlite_bad", { + columns: { + id: SQLite.nullable(SQLite.text), + both: SQLite.column(SQLite.text, { default: "a", defaultExpr: "('b')" }), + }, + primaryKey: ["id", "missing" as "id"], + indexes: [SQLite.index("bad_none", ["nope" as "id"])], + }) + const Clash = SQLite.table("clash", { columns: { id: SQLite.text }, indexes: [SQLite.index("short_link", ["id"])] }) + const exit = await Effect.runPromiseExit(sqliteEntitiesOf([Bad, Clash, ShortLink])) + if (exit._tag !== "Failure") throw new Error("a broken schema was accepted") + const messages = JSON.stringify(exit.cause) + for (const expected of ["starts with sqlite_", "cannot be nullable", "missing is not a column", "one of default, defaultExpr", "nope is not a column", "one namespace"]) { + expect(messages).toContain(expected) + } +}) + +it("refuses definitions SQLite would only reject when it runs the DDL", async () => { + // Each of these renders, then fails in SQLite: duplicate column name, a repeated key column, + // "default value of column is not constant", or an insert hitting NOT NULL. + const Bad = SQLite.table("bad_ddl", { + columns: { + id: SQLite.text, + Id: SQLite.text, + a: SQLite.column(SQLite.text, { name: "b" }), + b: SQLite.text, + copy: SQLite.column(SQLite.text, { defaultExpr: ($) => $.b }), + maybe: SQLite.column(SQLite.text, { default: undefined as never }), + stamped: SQLite.column(SQLite.timestamp, { defaultExpr: "(CURRENT_TIMESTAMP)" }), + epoch: SQLite.column(SQLite.timestamp, { defaultExpr: "(strftime('%s', 'now'))" }), + dsl: SQLite.column(SQLite.timestamp, { defaultExpr: () => SQLite.rawExpr("datetime('now')", SQLite.timestamp) }), + }, + primaryKey: ["id", "id"], + indexes: [SQLite.index("bad_ddl_twice", ["b", "b"])], + }) + const exit = await Effect.runPromiseExit(sqliteEntitiesOf([Bad])) + if (exit._tag !== "Failure") throw new Error("a broken schema was accepted") + const messages = JSON.stringify(exit.cause) + for (const expected of [ + "Id has the same SQL name as id", + "b has the same SQL name as a", + "id is listed more than once", + "b is listed more than once", + "defaultExpr reads b", + "default is undefined", + "bad_ddl.stamped", + "bad_ddl.epoch", + "bad_ddl.dsl", + "use SQLite.timestampNow", + ]) { + expect(messages).toContain(expected) + } +}) + +it("stamps rows with timestampNow in the form a timestamp column compares with", async () => { + const Events = SQLite.table("events", { + columns: { id: SQLite.integer, at: SQLite.column(SQLite.timestamp, { defaultExpr: SQLite.timestampNow }) }, + primaryKey: ["id"], + }) + const rows = await Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + for (const sql of yield* Effect.map(sqliteEntitiesOf([Events]), renderSqliteSchema)) yield* db.execute({ sql }) + yield* db.run(SQLite.insertInto(Events).values({ id: 1 })) + const hourAgo = DateTime.makeUnsafe(Date.now() - 3_600_000) + const inAnHour = DateTime.makeUnsafe(Date.now() + 3_600_000) + return yield* db.run(SQLite.from(Events).select("id").where(($) => [$.at.gt(hourAgo), $.at.lt(inAnHour)])) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + expect(rows).toEqual([{ id: 1 }]) +}) + +it("keeps entities as plain data that survive a JSON round trip, sorted the same every time", async () => { + const entities = await Effect.runPromise(sqliteEntitiesOf([Membership, ShortLink])) + const again = await Effect.runPromise(sqliteEntitiesOf([ShortLink, Membership])) + expect(canonicalJson(again)).toBe(canonicalJson(entities)) + const decoded = Schema.decodeUnknownSync(Schema.Array(SqliteSchemaEntity))(JSON.parse(canonicalJson(entities))) + expect(renderSqliteSchema(decoded)).toEqual(renderSqliteSchema(entities)) + expect(entities.map((e) => e.kind).filter((k, i, all) => all.indexOf(k) === i)).toEqual(["table", "column", "index", "unique"]) +}) diff --git a/src/sqlite/sqlite.test-d.ts b/src/sqlite/sqlite.test-d.ts index 7d3f78a..d4ad63f 100644 --- a/src/sqlite/sqlite.test-d.ts +++ b/src/sqlite/sqlite.test-d.ts @@ -20,3 +20,27 @@ const External = SQLite.table("ext", { external: true, columns: { id: SQLite.int SQLite.from(External).select("id") // @ts-expect-error an external table has only its declared columns SQLite.from(External).select("columns") + +const Links = SQLite.table("links", { + columns: { + id: SQLite.text, + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + hits: SQLite.column(SQLite.integer, { default: 0 }), + code: SQLite.column(SQLite.text, { unique: true }), + }, + primaryKey: ["id"], + indexes: [SQLite.index("links_org", ["orgId"])], +}) +expectTypeOf(Links.ddl.dialect).toEqualTypeOf<"sqlite">() +// A defaulted column may be left out of an insert; a nullable one too. +SQLite.insertInto(Links).values({ id: "a", code: "c" }) +// @ts-expect-error a unique column without a default is still required +SQLite.insertInto(Links).values({ id: "a" }) +// @ts-expect-error a primary key names declared columns only +SQLite.table("bad", { columns: { id: SQLite.text }, primaryKey: ["nope"] }) +// @ts-expect-error a default cannot be undefined: the column would be optional with no DEFAULT +SQLite.column(SQLite.text, { default: undefined }) +// @ts-expect-error a default is checked against the column's type +SQLite.table("bad", { columns: { n: SQLite.column(SQLite.integer, { default: "x" }) } }) +// @ts-expect-error an external table carries no DDL +External.ddl diff --git a/src/sqlite/types.ts b/src/sqlite/types.ts index 9fb9990..795e26c 100644 --- a/src/sqlite/types.ts +++ b/src/sqlite/types.ts @@ -2,7 +2,7 @@ import { Schema, SchemaGetter } from "effect" import { brand, custom, type CHType } from "../ch/types" import { SqliteTimestamp, SqliteTimestampLiteral } from "./timestamp" -export { sqliteTimestampText } from "./timestamp" +export { sqliteTimestampText, timestampNow } from "./timestamp" /** A SQLite storage type and the codec for its wire values. */ export type SqliteType = CHType diff --git a/tests/package-consumer-declarations.mts b/tests/package-consumer-declarations.mts index e4a191d..c0561d9 100644 --- a/tests/package-consumer-declarations.mts +++ b/tests/package-consumer-declarations.mts @@ -52,3 +52,8 @@ import * as SQLite from "@maple-dev/effect-orm/sqlite" export const SqliteItems = SQLite.table("sqlite_items", { id: SQLite.integer, label: SQLite.text }) export const sqliteById = SQLite.from(SqliteItems).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) export const sqliteCompile = () => SQLite.compile(sqliteById, { id: 1 }) +export const SqliteLinks = SQLite.table("sqlite_links", { + columns: { id: SQLite.text, hits: SQLite.column(SQLite.integer, { default: 0 }) }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("sqlite_links_hits", ["hits"])], +}) diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index 6c62a7b..fc6fc79 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -102,3 +102,12 @@ assert.equal(sqliteCompiled.dialect, "sqlite") const sqliteDecoded: ReadonlyArray<{ readonly id: number; readonly note: string }> = await Effect.runPromise(sqliteCompiled.decodeRows([{ id: 1, note: "sqlite" }])) assert.deepEqual(sqliteDecoded, [{ id: 1, note: "sqlite" }]) assert.equal(SQLite.compileUnsafe(sqliteQuery, { id: 1 }, { dialect: SQLite.d1Dialect }).dialect, "d1") +const sqliteLinks = SQLite.table("consumer_links", { + columns: { id: SQLite.text, orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }) }, + primaryKey: ["id"], + indexes: [SQLite.index("consumer_links_org", ["orgId"])], +}) +assert.deepEqual(await Effect.runPromise(Effect.map(S.sqliteEntitiesOf([sqliteLinks]), S.renderSqliteSchema)), [ + 'CREATE TABLE IF NOT EXISTS "consumer_links" (\n\t"id" text PRIMARY KEY NOT NULL,\n\t"org_id" text\n)', + 'CREATE INDEX IF NOT EXISTS "consumer_links_org" ON "consumer_links" ("org_id")', +])