From 283814676ae5a1f48c74736b96a815535b4e75c1 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:55:53 +0200 Subject: [PATCH 01/15] feat(sqlite): generate writes a first SQLite migration and snapshot --- src/kit/generate.ts | 60 ++++++++++++-------- src/kit/sqlite-generate.test.ts | 72 ++++++++++++++++++++++++ src/migrate/run.ts | 10 +++- src/migrate/source.ts | 16 ++++++ src/migrate/verify.ts | 4 +- src/schema/entities.ts | 22 +++++--- src/schema/snapshot.ts | 19 ++++++- src/schema/sqlite-diff.ts | 78 ++++++++++++++++++++++++++ src/schema/sqlite-ops.ts | 96 ++++++++++++++++++++++++++++++++ src/schema/sqlite-render.ts | 3 +- src/sqlite/ddl.test.ts | 21 +------ src/sqlite/shape.test-support.ts | 27 +++++++++ 12 files changed, 369 insertions(+), 59 deletions(-) create mode 100644 src/kit/sqlite-generate.test.ts create mode 100644 src/schema/sqlite-diff.ts create mode 100644 src/schema/sqlite-ops.ts create mode 100644 src/sqlite/shape.test-support.ts diff --git a/src/kit/generate.ts b/src/kit/generate.ts index 8d88bbb..bfed7ea 100644 --- a/src/kit/generate.ts +++ b/src/kit/generate.ts @@ -17,6 +17,10 @@ import { diffPgSchemas } from "../schema/pg-diff" import type { PgSchemaEntity } from "../schema/pg-entities" import { labelOfPg, renderPgOp, type PgMigrationOp } from "../schema/pg-ops" import type { RenderOptions } from "../schema/render" +import { sqliteEntitiesOf, type SqliteSchemaTable } from "../schema/sqlite-define" +import { diffSqliteSchemas } from "../schema/sqlite-diff" +import type { SqliteSchemaEntity } from "../schema/sqlite-entities" +import { labelOfSqlite, renderSqliteMigrationSql, renderSqliteOp, type SqliteMigrationOp } from "../schema/sqlite-ops" import { dialectOfObject, entitiesOf, @@ -31,7 +35,7 @@ import { analyze, type GraphProblem } from "./graph" export interface KitConfig { /** The database this folder migrates. Default `clickhouse`. */ readonly dialect?: SchemaDialect - /** Modules whose exports include `table` / `materializedView` values from `/clickhouse` or `/postgres`. */ + /** Modules whose exports include `table` / `materializedView` values from `/clickhouse`, `/postgres` or `/sqlite`. */ readonly schema: string | ReadonlyArray /** The migrations folder. */ readonly out: string @@ -43,6 +47,7 @@ export interface KitConfig { * What a generated Postgres migration holds. `ops` (default) writes `migration.json`; * `sql` writes the rendered statements to `migration.sql`, split by * `--> statement-breakpoint`, so a tool that applies drizzle-kit folders applies it too. + * A SQLite migration is always `migration.sql`. */ readonly emit?: "ops" | "sql" } @@ -193,6 +198,8 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption } let file: MigrationFile | undefined + /** A SQLite migration is SQL: there is no ops file for it. */ + let sqliteOps: ReadonlyArray | undefined let snapshotEntities: ReadonlyArray let prevIds = analysis.baseIds if (options.baseline === "drizzle") { @@ -212,7 +219,11 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption } else { const objects = yield* loadSchema(config, cwd) const entities: Effect.Effect, SchemaDefinitionError> = - dialect === "postgres" ? pgEntitiesOf(objects) : entitiesOf(objects) + dialect === "postgres" + ? pgEntitiesOf(objects) + : dialect === "sqlite" + ? sqliteEntitiesOf(objects as ReadonlyArray>) + : entitiesOf(objects) const next = yield* entities.pipe( Effect.mapError( (error) => @@ -228,10 +239,12 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption else if (options.custom === true) snapshotEntities = analysis.base else { const hints = [...(options.hints ?? [])] - const diff = (): DiffResult => + const diff = (): DiffResult => dialect === "postgres" ? diffPgSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) - : diffSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) + : dialect === "sqlite" + ? diffSqliteSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) + : diffSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) let result = diff() if (result.unsupported.length > 0) { return yield* new KitError({ @@ -254,10 +267,9 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption }) } if (result.ops.length === 0) return { written: undefined, plan: [] } satisfies GenerateResult - file = - dialect === "postgres" - ? { version: "1", dialect: "postgres", ops: result.ops as ReadonlyArray } - : { version: "1", ops: result.ops as ReadonlyArray } + if (dialect === "sqlite") sqliteOps = result.ops as ReadonlyArray + else if (dialect === "postgres") file = { version: "1", dialect: "postgres", ops: result.ops as ReadonlyArray } + else file = { version: "1", ops: result.ops as ReadonlyArray } } } @@ -269,29 +281,31 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption // Written aside and renamed into place, so a reader never sees half a migration. yield* Effect.gen(function* () { yield* io(() => mkdir(staging, { recursive: true }), `Cannot create ${staging}`) - yield* io( - () => - file === undefined - ? writeFile( - join(staging, "migration.sql"), + const [fileName, contents] = + sqliteOps !== undefined + ? ["migration.sql", renderSqliteMigrationSql(sqliteOps, STATEMENT_BREAKPOINT)] + : file === undefined + ? [ + "migration.sql", options.baseline !== undefined ? "-- Baseline: the schema as it stood when effect-orm took over this folder. Runs nothing.\n" : "-- Custom SQL migration. Separate statements with a line holding only:\n-- --> statement-breakpoint\n", - ) + ] : "dialect" in file && config.emit === "sql" - ? writeFile(join(staging, "migration.sql"), renderPgSqlFile(file.ops)) - : writeFile(join(staging, "migration.json"), `${JSON.stringify(file, null, "\t")}\n`), - `Cannot write ${dir}`, - ) + ? ["migration.sql", renderPgSqlFile(file.ops)] + : ["migration.json", `${JSON.stringify(file, null, "\t")}\n`] + yield* io(() => writeFile(join(staging, fileName), contents), `Cannot write ${dir}`) yield* io(() => writeFile(join(staging, "snapshot.json"), serializeSnapshot(snapshot)), `Cannot write ${dir}`) yield* io(() => rename(staging, dir), `Cannot move ${staging} to ${dir}`) }).pipe(Effect.onError(() => Effect.promise(() => rm(staging, { recursive: true, force: true }).catch(() => undefined)))) const plan = - file === undefined - ? [] - : "dialect" in file - ? file.ops.map((op) => ({ label: labelOfPg(op), sql: renderPgOp(op) })) - : file.ops.map((op) => ({ label: labelOf(op), sql: renderOp(op, config.render) })) + sqliteOps !== undefined + ? sqliteOps.map((op) => ({ label: labelOfSqlite(op), sql: renderSqliteOp(op) })) + : file === undefined + ? [] + : "dialect" in file + ? file.ops.map((op) => ({ label: labelOfPg(op), sql: renderPgOp(op) })) + : file.ops.map((op) => ({ label: labelOf(op), sql: renderOp(op, config.render) })) return { written: dir, plan } satisfies GenerateResult }) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts new file mode 100644 index 0000000..9752470 --- /dev/null +++ b/src/kit/sqlite-generate.test.ts @@ -0,0 +1,72 @@ +// `generate` for SQLite, checked on real SQLite: the migrations it writes, +// applied in order, must give the same PRAGMA results as creating the final +// schema fresh. +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { Effect } from "effect" +import { afterEach, beforeEach, expect, it } from "vitest" +import { generate, type GenerateOptions, type KitConfig } from "../kit" +import { STATEMENT_BREAKPOINT } from "../migrate/source" +import { renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaTable } from "../schema" +import { shapeAfter } from "../sqlite/shape.test-support" + +const src = resolve(import.meta.dirname, "..") +let dir = "" +let version = 0 + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "effect-orm-sqlite-")) + version = 0 +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +/** Write a schema module (a fresh path each time, so the import cache never hands back an old one) and run generate on it. */ +const generateFrom = (body: string, options: GenerateOptions = {}) => { + const file = `schema${++version}.ts` + writeFileSync(join(dir, file), `import * as SQLite from "${src}/sqlite"\n${body}\n`) + const config: KitConfig = { dialect: "sqlite", schema: `./${file}`, out: "./migrations" } + return Effect.runPromise(generate(config, dir, { name: `v${version}`, ...options })) +} + +const folders = () => readdirSync(join(dir, "migrations")).sort() +const migrationSql = (folder: string) => readFileSync(join(dir, "migrations", folder, "migration.sql"), "utf8") + +/** Every statement of every migration, in folder order, split as Alchemy and drizzle split them. */ +const allStatements = () => + folders().flatMap((folder) => + migrationSql(folder) + .split(STATEMENT_BREAKPOINT) + .map((s) => s.trim()) + .filter((s) => s.replace(/^\s*--.*$/gm, "").trim().length > 0), + ) + +/** The schema module's tables, created fresh. */ +const freshShape = async (body: string) => { + const file = `fresh${++version}.ts` + writeFileSync(join(dir, file), `import * as SQLite from "${src}/sqlite"\n${body}\n`) + const module = (await import(join(dir, file))) as Record> + return shapeAfter(await Effect.runPromise(Effect.map(sqliteEntitiesOf(Object.values(module)), renderSqliteSchema))) +} + +const expectMigratedToEqualFresh = async (body: string) => expect(await shapeAfter(allStatements())).toEqual(await freshShape(body)) + +const links = (columns: string, rest = "") => ` +export const Links = SQLite.table("links", { + columns: { ${columns} }, + primaryKey: ["id"], + ${rest} +})` + +it("writes a first migration as migration.sql plus a sqlite snapshot, then nothing for an unchanged schema", async () => { + const v1 = links(`id: SQLite.text, url: SQLite.text`) + const first = await generateFrom(v1) + expect(first.plan.map((p) => p.label)).toEqual(["metadata"]) + const [folder] = folders() + expect(folder).toMatch(/^\d{14}_v1$/) + expect(readdirSync(join(dir, "migrations", folder!)).sort()).toEqual(["migration.sql", "snapshot.json"]) + expect(JSON.parse(readFileSync(join(dir, "migrations", folder!, "snapshot.json"), "utf8")).dialect).toBe("sqlite") + expect(migrationSql(folder!)).toContain('CREATE TABLE IF NOT EXISTS "links"') + await expectMigratedToEqualFresh(v1) + expect((await generateFrom(v1)).written).toBeUndefined() +}) diff --git a/src/migrate/run.ts b/src/migrate/run.ts index 3b7efdf..e39bdbd 100644 --- a/src/migrate/run.ts +++ b/src/migrate/run.ts @@ -27,7 +27,7 @@ import { } from "./ledger" import { ensurePgLedger, isPgApplied, lockPg, readPgApplied, recordPgMigration, transactionOf } from "./pg-ledger" import { backfillWindows, renderBackfill, renderBackfillBounds, renderOp } from "../schema/ops" -import { dialectOf, stepsOf, type LoadedMigration, type MigrationStep } from "./source" +import { dialectOf, refuseSqlite, stepsOf, type LoadedMigration, type MigrationStep } from "./source" export interface RunOptions { readonly migrations: ReadonlyArray @@ -49,6 +49,7 @@ export interface RunOptions { const resolveDialect = (migrations: ReadonlyArray, dialect: SchemaDialect | undefined): SchemaDialect => dialect ?? dialectOf(migrations) ?? "clickhouse" + export interface AppliedMigration { readonly name: string readonly steps: number @@ -250,7 +251,9 @@ const applyOne = (migration: LoadedMigration, render: RenderOptions, renew: Effe * have their hash checked. Returns what ran. */ export const run = (options: RunOptions): Effect.Effect, MigrateError, MigrationDriver> => - resolveDialect(options.migrations, options.dialect) === "postgres" ? runPg(options) : runClickHouse(options) + Effect.andThen(refuseSqlite(options.migrations, options.dialect), () => + resolveDialect(options.migrations, options.dialect) === "postgres" ? runPg(options) : runClickHouse(options), + ) const runClickHouse = (options: RunOptions): Effect.Effect, MigrateError, MigrationDriver> => Effect.gen(function* () { @@ -335,6 +338,7 @@ export const baseline = (options: BaselineOptions): Effect.Effect m.name === options.upTo) if (index === -1) return yield* new MigrateSourceError({ migration: options.upTo, message: "is not one of the migrations" }) + yield* refuseSqlite(options.migrations, options.dialect) const postgres = resolveDialect(options.migrations, options.dialect) === "postgres" if (postgres) yield* ensurePgLedger else yield* ensureLedger(options.render ?? {}) @@ -355,6 +359,7 @@ export const status = ( dialect?: SchemaDialect, ): Effect.Effect, MigrateError, MigrationDriver> => Effect.gen(function* () { + yield* refuseSqlite(migrations, dialect) if (resolveDialect(migrations, dialect) === "postgres") { yield* ensurePgLedger const applied = new Map((yield* readPgApplied).map((row) => [row.name, row])) @@ -402,6 +407,7 @@ export interface ResolveStepOptions { export const resolveStep = (options: ResolveStepOptions): Effect.Effect => Effect.gen(function* () { const { migration } = options + yield* refuseSqlite([migration], options.dialect) if (resolveDialect([migration], options.dialect) === "postgres") { return yield* new MigrateSourceError({ migration: migration.name, diff --git a/src/migrate/source.ts b/src/migrate/source.ts index 11e1e16..16cec6b 100644 --- a/src/migrate/source.ts +++ b/src/migrate/source.ts @@ -216,3 +216,19 @@ export const dialectOf = (migrations: ReadonlyArray): SchemaDia } return undefined } + +/** + * `generate` writes SQLite migrations, but this runner does not apply them yet: + * apply the folder with a tool that reads drizzle-kit folders (Alchemy's D1 + * resource, drizzle's migrator). Refused here rather than run with the + * ClickHouse ledger. + */ +export const refuseSqlite = (migrations: ReadonlyArray, dialect: SchemaDialect | undefined) => + (dialect ?? dialectOf(migrations)) === "sqlite" + ? Effect.fail( + new MigrateSourceError({ + migration: migrations[0]?.name ?? "(none)", + message: "SQLite migrations are not run by effect-orm yet; apply the folder with a tool that reads drizzle-kit folders", + }), + ) + : Effect.void diff --git a/src/migrate/verify.ts b/src/migrate/verify.ts index b5b02d2..1ee74d0 100644 --- a/src/migrate/verify.ts +++ b/src/migrate/verify.ts @@ -19,7 +19,7 @@ import type { MigrateSourceError, MigrateSqlError } from "./errors" import { LEDGER_TABLES, readApplied } from "./ledger" import { readPgApplied } from "./pg-ledger" import { verifyPg, type PgVerifyOptions } from "./pg-verify" -import { dialectOf, type LoadedMigration } from "./source" +import { dialectOf, refuseSqlite, type LoadedMigration } from "./source" export interface Drift { readonly entity: string @@ -116,6 +116,7 @@ export const verify = ( options: VerifyOptions = {}, ): Effect.Effect => Effect.gen(function* () { + yield* refuseSqlite(migrations, options.dialect) const postgres = (options.dialect ?? dialectOf(migrations)) === "postgres" const appliedRows = yield* (postgres ? readPgApplied : readApplied).pipe(Effect.orElseSucceed(() => [])) const appliedNames = new Set(appliedRows.map((row) => row.name)) @@ -123,6 +124,7 @@ export const verify = ( const snapshot = against?.snapshot if (against === undefined || snapshot === undefined) return { against: undefined, drift: [] } if (snapshot.dialect === "postgres") return { against: against.name, drift: yield* verifyPg(snapshot, against.name, options) } + if (snapshot.dialect === "sqlite") return yield* refuseSqlite(migrations, "sqlite").pipe(Effect.as({ against: undefined, drift: [] })) return { against: against.name, drift: yield* verifyClickHouse(snapshot) } }) diff --git a/src/schema/entities.ts b/src/schema/entities.ts index a625276..5b4dc04 100644 --- a/src/schema/entities.ts +++ b/src/schema/entities.ts @@ -9,7 +9,7 @@ import { Schema } from "effect" import { PgSchemaEntity } from "./pg-entities" -import type { SqliteSchemaEntity } from "./sqlite-entities" +import { SqliteSchemaEntity } from "./sqlite-entities" /** `DEFAULT`, `MATERIALIZED`, or `ALIAS`, with its SQL expression. */ export const ColumnDefault = Schema.Struct({ @@ -103,21 +103,25 @@ export const PgSnapshot = Schema.Struct({ }) export type PgSnapshot = typeof PgSnapshot.Type +export const SqliteSnapshot = Schema.Struct({ + ...snapshotFields, + dialect: Schema.Literal("sqlite"), + entities: Schema.Array(SqliteSchemaEntity), +}) +export type SqliteSnapshot = typeof SqliteSnapshot.Type + /** The schema at one point in history. `dialect` says which entity set it holds. */ -export const Snapshot = Schema.Union([ClickHouseSnapshot, PgSnapshot]) +export const Snapshot = Schema.Union([ClickHouseSnapshot, PgSnapshot, SqliteSnapshot]) export type Snapshot = typeof Snapshot.Type /** The dialects a schema can be written for. */ export type SchemaDialect = Snapshot["dialect"] -/** An entity of either dialect. Everything dialect-neutral (keys, ordering, hashing, the branch graph) takes this. */ -export type AnySchemaEntity = SchemaEntity | PgSchemaEntity - -/** What `entityKey` and `sortEntities` take: a snapshot dialect's entity, or a SQLite one (no snapshot yet). */ -type KeyedEntity = AnySchemaEntity | SqliteSchemaEntity +/** An entity of any dialect. Everything dialect-neutral (keys, ordering, hashing, the branch graph) takes this. */ +export type AnySchemaEntity = SchemaEntity | PgSchemaEntity | SqliteSchemaEntity /** A stable key per entity, unique within a snapshot: `:`, or `:.` for a table's parts. */ -export const entityKey = (entity: KeyedEntity): string => +export const entityKey = (entity: AnySchemaEntity): string => "table" in entity ? `${entity.kind}:${entity.table}.${entity.name}` : entity.kind === "enum" && entity.schema !== null @@ -135,7 +139,7 @@ const kindOrder: Record = { } /** Enums, tables, then columns in declaration order, then indexes, unique constraints, foreign keys, views. Deterministic. */ -export const sortEntities = (entities: ReadonlyArray): ReadonlyArray => +export const sortEntities = (entities: ReadonlyArray): ReadonlyArray => [...entities].sort((a, b) => { const byKind = kindOrder[a.kind] - kindOrder[b.kind] if (byKind !== 0) return byKind diff --git a/src/schema/snapshot.ts b/src/schema/snapshot.ts index 181d249..5495775 100644 --- a/src/schema/snapshot.ts +++ b/src/schema/snapshot.ts @@ -15,12 +15,20 @@ import { type SchemaDialect, type SchemaEntity, type Snapshot, + type SqliteSnapshot, } from "./entities" import { enumProblems, type PgEnum, type PgSchemaTable } from "./pg-define" import type { PgEnumEntity, PgSchemaEntity } from "./pg-entities" +import type { SqliteSchemaTable } from "./sqlite-define" +import type { SqliteSchemaEntity } from "./sqlite-entities" /** Anything a schema module may export that `generate` collects. */ -export type SchemaObject = SchemaTable | MaterializedView | PgSchemaTable | PgEnum +export type SchemaObject = + | SchemaTable + | MaterializedView + | PgSchemaTable + | PgEnum + | SqliteSchemaTable export const isSchemaObject = (value: unknown): value is SchemaObject => typeof value === "object" && @@ -47,7 +55,7 @@ export const entitiesOf = (objects: ReadonlyArray): Effect.Effect< for (const object of objects) { problems.push(...object.problems) if (dialectOfObject(object) !== "clickhouse") { - problems.push({ object: object.name, message: "is a Postgres table; use pgEntitiesOf" }) + problems.push({ object: object.name, message: `is a ${dialectOfObject(object) === "postgres" ? "Postgres" : "SQLite"} table; use ${dialectOfObject(object) === "postgres" ? "pgEntitiesOf" : "sqliteEntitiesOf"}` }) continue } if (object._tag === "Table") { @@ -101,7 +109,7 @@ export const pgEntitiesOf = (objects: ReadonlyArray): Effect.Effec for (const object of objects) { if (dialectOfObject(object) !== "postgres") { problems.push(...object.problems) - problems.push({ object: object.name, message: "is a ClickHouse definition; use entitiesOf" }) + problems.push({ object: object.name, message: dialectOfObject(object) === "sqlite" ? "is a SQLite table; use sqliteEntitiesOf" : "is a ClickHouse definition; use entitiesOf" }) continue } if (object._tag === "PgEnum") { @@ -175,6 +183,11 @@ export function makeSnapshot( prevIds: ReadonlyArray, dialect: "postgres", ): Effect.Effect +export function makeSnapshot( + entities: ReadonlyArray, + prevIds: ReadonlyArray, + dialect: "sqlite", +): Effect.Effect export function makeSnapshot( entities: ReadonlyArray, prevIds: ReadonlyArray, diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts new file mode 100644 index 0000000..ade1e7d --- /dev/null +++ b/src/schema/sqlite-diff.ts @@ -0,0 +1,78 @@ +// Diff two SQLite schemas into migration ops. +// +// Pure and offline, like `diffPgSchemas`, with the same data-loss rule: +// dropping a table or a column needs a `confirm_data_loss` hint. + +import { canonicalJson, entityKey } from "./entities" +import { hintKey, type DiffResult, type Hint } from "./diff" +import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteUniqueEntity } from "./sqlite-entities" +import type { SqliteMigrationOp } from "./sqlite-ops" + +const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonicalJson(b) + +const pick = (entities: ReadonlyArray, kind: K) => + entities.filter((e): e is Extract => e.kind === kind) + +const opOrder: Record = { + drop_index: 0, + drop_table: 1, + create_table: 2, + create_index: 3, + rename_table: 0, + rename_column: 0, + add_column: 0, + drop_column: 0, + rebuild_table: 0, +} + +export const diffSqliteSchemas = ( + prev: ReadonlyArray, + next: ReadonlyArray, + hints: ReadonlyArray = [], +): DiffResult => { + const given = new Set(hints.map(hintKey)) + const ops: Array = [] + const missingHints: Array = [] + const confirm = (hint: Hint, op: SqliteMigrationOp): void => { + if (given.has(hintKey(hint))) ops.push(op) + else missingHints.push(hint) + } + + const tablesOf = (entities: ReadonlyArray) => new Map(pick(entities, "table").map((t) => [t.name, t])) + const prevTables = tablesOf(prev) + const nextTables = tablesOf(next) + const partsOf = (list: ReadonlyArray, table: string) => + list.filter((e) => e.table === table) + const columns = (entities: ReadonlyArray, table: string) => + partsOf(pick(entities, "column"), table).sort((a, b) => a.position - b.position) + + for (const [name, table] of nextTables) { + if (!prevTables.has(name)) ops.push({ op: "create_table", table, columns: columns(next, name), uniques: partsOf(pick(next, "unique"), name) }) + } + for (const name of prevTables.keys()) { + if (!nextTables.has(name)) confirm({ type: "confirm_data_loss", kind: "table", entity: name }, { op: "drop_table", name }) + } + + const prevIndexes = new Map(pick(prev, "index").map((i) => [entityKey(i), i])) + const nextIndexes = new Map(pick(next, "index").map((i) => [entityKey(i), i])) + for (const [key, index] of nextIndexes) { + const old = prevIndexes.get(key) + if (old !== undefined && same(old, index)) continue + if (old !== undefined) ops.push({ op: "drop_index", name: old.name }) + ops.push({ op: "create_index", index }) + } + for (const [key, index] of prevIndexes) { + // A dropped table takes its indexes with it. + if (!nextIndexes.has(key) && nextTables.has(index.table)) ops.push({ op: "drop_index", name: index.name }) + } + + return { + ops: ops + .map((op, i) => [op, i] as const) + .sort(([a, i], [b, j]) => opOrder[a.op] - opOrder[b.op] || i - j) + .map(([op]) => op), + missingHints, + unsupported: [], + } +} + diff --git a/src/schema/sqlite-ops.ts b/src/schema/sqlite-ops.ts new file mode 100644 index 0000000..241f861 --- /dev/null +++ b/src/schema/sqlite-ops.ts @@ -0,0 +1,96 @@ +// SQLite migration operations and the SQL they render to. +// +// SQLite's ALTER TABLE can rename a table or a column, add a column and drop +// one, and little else. Any other change (a column's type, nullability or +// default, the primary key, a UNIQUE constraint) is a table rebuild: the +// procedure https://www.sqlite.org/lang_altertable.html#otheralter describes, +// as drizzle-kit writes it. Create the new table under a temporary name, copy +// the rows, drop the old table, rename the new one into place, and create its +// indexes again (dropping the table dropped them). + +import { Schema } from "effect" +import { SqliteColumnEntity, SqliteIndexEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" +import { renderSqliteColumnDefinition, renderSqliteCreateIndex, renderSqliteCreateTable, sqliteIdent } from "./sqlite-render" + +export const SqliteMigrationOp = Schema.Union([ + Schema.Struct({ + op: Schema.Literal("create_table"), + table: SqliteTableEntity, + columns: Schema.Array(SqliteColumnEntity), + uniques: Schema.Array(SqliteUniqueEntity), + }), + Schema.Struct({ op: Schema.Literal("drop_table"), name: Schema.String }), + Schema.Struct({ op: Schema.Literal("rename_table"), from: Schema.String, to: Schema.String }), + Schema.Struct({ op: Schema.Literal("add_column"), column: SqliteColumnEntity }), + Schema.Struct({ op: Schema.Literal("drop_column"), table: Schema.String, name: Schema.String }), + Schema.Struct({ op: Schema.Literal("rename_column"), table: Schema.String, from: Schema.String, to: Schema.String }), + /** The table as it will be, and the columns whose rows carry over (by their name in both tables). */ + Schema.Struct({ + op: Schema.Literal("rebuild_table"), + table: SqliteTableEntity, + columns: Schema.Array(SqliteColumnEntity), + uniques: Schema.Array(SqliteUniqueEntity), + copy: Schema.Array(Schema.String), + }), + Schema.Struct({ op: Schema.Literal("create_index"), index: SqliteIndexEntity }), + Schema.Struct({ op: Schema.Literal("drop_index"), name: Schema.String }), +]) +export type SqliteMigrationOp = typeof SqliteMigrationOp.Type + +/** Labels the plan prints. `rebuild`: the table is copied into a new one. */ +export type SqliteOpLabel = "metadata" | "destructive" | "rebuild" + +export const labelOfSqlite = (op: SqliteMigrationOp): SqliteOpLabel => { + switch (op.op) { + case "drop_table": + case "drop_column": + return "destructive" + case "rebuild_table": + return "rebuild" + default: + return "metadata" + } +} + +/** The temporary name a rebuilt table is created under, drizzle-kit's. */ +export const rebuildName = (table: string): string => `__new_${table}` + +const list = (names: ReadonlyArray): string => names.map(sqliteIdent).join(", ") + +/** The statements one op runs, in order. */ +export const renderSqliteOp = (op: SqliteMigrationOp): ReadonlyArray => { + switch (op.op) { + case "create_table": + return [renderSqliteCreateTable(op.table, op.columns, op.uniques)] + case "drop_table": + return [`DROP TABLE IF EXISTS ${sqliteIdent(op.name)}`] + case "rename_table": + return [`ALTER TABLE ${sqliteIdent(op.from)} RENAME TO ${sqliteIdent(op.to)}`] + case "add_column": + return [`ALTER TABLE ${sqliteIdent(op.column.table)} ADD COLUMN ${renderSqliteColumnDefinition(op.column)}`] + case "drop_column": + return [`ALTER TABLE ${sqliteIdent(op.table)} DROP COLUMN ${sqliteIdent(op.name)}`] + case "rename_column": + return [`ALTER TABLE ${sqliteIdent(op.table)} RENAME COLUMN ${sqliteIdent(op.from)} TO ${sqliteIdent(op.to)}`] + case "rebuild_table": { + const temporary = rebuildName(op.table.name) + const created = renderSqliteCreateTable({ ...op.table, name: temporary }, op.columns.map((c) => ({ ...c, table: temporary })), op.uniques, { ifNotExists: false }) + return [ + "PRAGMA foreign_keys=OFF", + created, + ...(op.copy.length === 0 ? [] : [`INSERT INTO ${sqliteIdent(temporary)}(${list(op.copy)}) SELECT ${list(op.copy)} FROM ${sqliteIdent(op.table.name)}`]), + `DROP TABLE ${sqliteIdent(op.table.name)}`, + `ALTER TABLE ${sqliteIdent(temporary)} RENAME TO ${sqliteIdent(op.table.name)}`, + "PRAGMA foreign_keys=ON", + ] + } + case "create_index": + return [renderSqliteCreateIndex(op.index)] + case "drop_index": + return [`DROP INDEX IF EXISTS ${sqliteIdent(op.name)}`] + } +} + +/** A migration.sql: each op's statements under a comment naming its label, one statement per breakpoint. */ +export const renderSqliteMigrationSql = (ops: ReadonlyArray, breakpoint: string): string => + `${ops.flatMap((op) => renderSqliteOp(op).map((statement) => `-- ${labelOfSqlite(op)}\n${statement};`)).join(`\n${breakpoint}\n`)}\n` diff --git a/src/schema/sqlite-render.ts b/src/schema/sqlite-render.ts index 20f871c..f0279eb 100644 --- a/src/schema/sqlite-render.ts +++ b/src/schema/sqlite-render.ts @@ -25,6 +25,7 @@ export const renderSqliteCreateTable = ( table: SqliteTableEntity, columns: ReadonlyArray, uniques: ReadonlyArray = [], + options: { readonly ifNotExists?: boolean } = {}, ): string => { const key = table.primaryKey?.columns ?? [] const inline = key.length === 1 ? key[0] : undefined @@ -34,7 +35,7 @@ export const renderSqliteCreateTable = ( .map((column) => renderSqliteColumnDefinition(column, column.name === inline, inlineUnique.has(column.name))) if (key.length > 1) body.push(`PRIMARY KEY (${list(key)})`) for (const unique of uniques) if (unique.columns.length > 1) body.push(`UNIQUE (${list(unique.columns)})`) - return `CREATE TABLE IF NOT EXISTS ${sqliteIdent(table.name)} (\n\t${body.join(",\n\t")}\n)` + return `CREATE TABLE ${options.ifNotExists === false ? "" : "IF NOT EXISTS "}${sqliteIdent(table.name)} (\n\t${body.join(",\n\t")}\n)` } export const renderSqliteCreateIndex = (index: SqliteIndexEntity): string => diff --git a/src/sqlite/ddl.test.ts b/src/sqlite/ddl.test.ts index 2f9516a..cdde870 100644 --- a/src/sqlite/ddl.test.ts +++ b/src/sqlite/ddl.test.ts @@ -5,6 +5,7 @@ import { expect, it } from "vitest" import * as Db from "../database" import { canonicalJson, renderSqliteSchema, SqliteSchemaEntity, sqliteEntitiesOf } from "../schema" import * as SQLite from "../sqlite" +import { shapeAfter } from "./shape.test-support" // drizzle-kit 0.31.11 (drizzle-orm 0.45.4) `generate` output for the same table. const drizzleShortLink = `CREATE TABLE \`short_link\` ( @@ -56,26 +57,6 @@ const ShortLink = SQLite.table("short_link", { const drizzleStatements = (sql: string) => sql.split("--> statement-breakpoint").map((s) => s.trim()).filter((s) => s.length > 0) -/** What SQLite reports about every table and index after running `statements` on an empty database. */ -const shapeAfter = (statements: ReadonlyArray) => - Effect.runPromise(Effect.gen(function* () { - const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) - for (const sql of statements) yield* db.execute({ sql }) - const tables = yield* db.query({ sql: "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name" }) - const shape: Record = {} - for (const { name } of tables as ReadonlyArray<{ name: string }>) { - const columns = yield* db.query({ sql: `SELECT * FROM pragma_table_info('${name}')` }) - const list = yield* db.query({ sql: `SELECT name, "unique", origin, partial FROM pragma_index_list('${name}') ORDER BY name` }) - const indexes = [] - for (const index of list as ReadonlyArray<{ name: string }>) { - const keys = yield* db.query({ sql: `SELECT seqno, cid, name, "desc", coll, "key" FROM pragma_index_xinfo('${index.name}')` }) - indexes.push({ ...index, keys }) - } - shape[name] = { columns, indexes } - } - return shape - }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) - const ddlOf = (tables: Parameters[0]) => Effect.runPromise(Effect.map(sqliteEntitiesOf(tables), renderSqliteSchema)) it("creates short_link exactly as drizzle-kit's SQL does, by PRAGMA table_info and index_list", async () => { diff --git a/src/sqlite/shape.test-support.ts b/src/sqlite/shape.test-support.ts new file mode 100644 index 0000000..72397da --- /dev/null +++ b/src/sqlite/shape.test-support.ts @@ -0,0 +1,27 @@ +// Test support: what SQLite reports about a schema, to compare two ways of building it. +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as Db from "../database" +import { sqliteDialect } from "./dialect" + +/** What SQLite reports about every table and index after running `statements` on an empty database. */ +export const shapeAfter = (statements: ReadonlyArray) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: sqliteDialect }) + for (const sql of statements) yield* db.execute({ sql }) + const tables = yield* db.query({ sql: "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name" }) + const shape: Record = {} + for (const { name } of tables as ReadonlyArray<{ name: string }>) { + const columns = yield* db.query({ sql: `SELECT * FROM pragma_table_info('${name}')` }) + const list = yield* db.query({ sql: `SELECT name, "unique", origin, partial FROM pragma_index_list('${name}') ORDER BY name` }) + const indexes = [] + for (const index of list as ReadonlyArray<{ name: string }>) { + const keys = yield* db.query({ sql: `SELECT seqno, cid, name, "desc", coll, "key" FROM pragma_index_xinfo('${index.name}')` }) + indexes.push({ ...index, keys }) + } + shape[name] = { columns, indexes } + } + return shape + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + From 4a0670d2553a0033866ab6f262b4758bc79c0159 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:56:48 +0200 Subject: [PATCH 02/15] feat(sqlite): add columns with ALTER TABLE ADD COLUMN --- src/kit/sqlite-generate.test.ts | 11 ++++ src/schema/sqlite-diff.ts | 113 +++++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index 9752470..4140808 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -70,3 +70,14 @@ it("writes a first migration as migration.sql plus a sqlite snapshot, then nothi await expectMigratedToEqualFresh(v1) expect((await generateFrom(v1)).written).toBeUndefined() }) + +it("adds a nullable column, and a NOT NULL one with a literal default, with ALTER TABLE ADD COLUMN", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const v2 = links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text), hits: SQLite.column(SQLite.integer, { default: 0 })`) + const second = await generateFrom(v2) + expect(second.plan.flatMap((p) => p.sql)).toEqual([ + 'ALTER TABLE "links" ADD COLUMN "note" text', + 'ALTER TABLE "links" ADD COLUMN "hits" integer NOT NULL DEFAULT 0', + ]) + await expectMigratedToEqualFresh(v2) +}) diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts index ade1e7d..574cb83 100644 --- a/src/schema/sqlite-diff.ts +++ b/src/schema/sqlite-diff.ts @@ -5,7 +5,7 @@ import { canonicalJson, entityKey } from "./entities" import { hintKey, type DiffResult, type Hint } from "./diff" -import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteUniqueEntity } from "./sqlite-entities" +import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" import type { SqliteMigrationOp } from "./sqlite-ops" const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonicalJson(b) @@ -13,16 +13,19 @@ const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonical const pick = (entities: ReadonlyArray, kind: K) => entities.filter((e): e is Extract => e.kind === kind) +// Indexes go first, so no dropped column is still indexed; new tables and +// columns exist before the indexes that need them; a rebuilt table's indexes +// are created after it. const opOrder: Record = { - drop_index: 0, - drop_table: 1, - create_table: 2, - create_index: 3, rename_table: 0, - rename_column: 0, - add_column: 0, - drop_column: 0, - rebuild_table: 0, + rename_column: 1, + drop_index: 2, + drop_column: 3, + drop_table: 4, + create_table: 5, + add_column: 6, + rebuild_table: 7, + create_index: 8, } export const diffSqliteSchemas = ( @@ -33,9 +36,11 @@ export const diffSqliteSchemas = ( const given = new Set(hints.map(hintKey)) const ops: Array = [] const missingHints: Array = [] - const confirm = (hint: Hint, op: SqliteMigrationOp): void => { - if (given.has(hintKey(hint))) ops.push(op) - else missingHints.push(hint) + /** Push `op` once every hint it needs was given; otherwise report the missing ones. */ + const confirm = (needed: Hint | ReadonlyArray, op: SqliteMigrationOp): void => { + const missing = (Array.isArray(needed) ? needed : [needed]).filter((hint) => !given.has(hintKey(hint))) + if (missing.length === 0) ops.push(op) + else missingHints.push(...missing) } const tablesOf = (entities: ReadonlyArray) => new Map(pick(entities, "table").map((t) => [t.name, t])) @@ -53,17 +58,30 @@ export const diffSqliteSchemas = ( if (!nextTables.has(name)) confirm({ type: "confirm_data_loss", kind: "table", entity: name }, { op: "drop_table", name }) } + for (const [name, after] of nextTables) { + const before = prevTables.get(name) + if (before === undefined) continue + diffTable( + { table: before, columns: columns(prev, name), uniques: partsOf(pick(prev, "unique"), name) }, + { table: after, columns: columns(next, name), uniques: partsOf(pick(next, "unique"), name) }, + ops, + confirm, + ) + } + const rebuilt = new Set(ops.flatMap((op) => (op.op === "rebuild_table" ? [op.table.name] : []))) + const prevIndexes = new Map(pick(prev, "index").map((i) => [entityKey(i), i])) const nextIndexes = new Map(pick(next, "index").map((i) => [entityKey(i), i])) for (const [key, index] of nextIndexes) { const old = prevIndexes.get(key) - if (old !== undefined && same(old, index)) continue - if (old !== undefined) ops.push({ op: "drop_index", name: old.name }) + // A rebuilt table lost its indexes with the old table; all of them are created again. + if (old !== undefined && same(old, index) && !rebuilt.has(index.table)) continue + if (old !== undefined && !rebuilt.has(index.table)) ops.push({ op: "drop_index", name: old.name }) ops.push({ op: "create_index", index }) } for (const [key, index] of prevIndexes) { // A dropped table takes its indexes with it. - if (!nextIndexes.has(key) && nextTables.has(index.table)) ops.push({ op: "drop_index", name: index.name }) + if (!nextIndexes.has(key) && nextTables.has(index.table) && !rebuilt.has(index.table)) ops.push({ op: "drop_index", name: index.name }) } return { @@ -76,3 +94,68 @@ export const diffSqliteSchemas = ( } } + +interface TableState { + readonly table: SqliteTableEntity + readonly columns: ReadonlyArray + readonly uniques: ReadonlyArray +} + +/** What a column is, apart from its place in the table. */ +const definition = (column: SqliteColumnEntity) => ({ type: column.type, notNull: column.notNull, default: column.default }) + +/** + * Can `ALTER TABLE ADD COLUMN` add it? Not to a key or a UNIQUE constraint, + * not NOT NULL without a default, and not with an expression default, which + * SQLite refuses there. + */ +const addable = (column: SqliteColumnEntity, after: TableState): boolean => + !(after.table.primaryKey?.columns ?? []).includes(column.name) && + !after.uniques.some((u) => u.columns.includes(column.name)) && + (!column.notNull || (column.default !== null && column.default.toUpperCase() !== "NULL")) && + (column.default === null || !/^\s*\(|^\s*current_(time|date|timestamp)\b/i.test(column.default)) + +/** Can `ALTER TABLE DROP COLUMN` drop it? Not from a key or a UNIQUE constraint. */ +const droppable = (name: string, before: TableState): boolean => + !(before.table.primaryKey?.columns ?? []).includes(name) && !before.uniques.some((u) => u.columns.includes(name)) + +/** + * One table's changes: ALTER TABLE where SQLite allows it, otherwise a + * rebuild. A rebuild copies every column the two tables share. + */ +const diffTable = ( + before: TableState, + after: TableState, + ops: Array, + confirm: (needed: Hint | ReadonlyArray, op: SqliteMigrationOp) => void, +): void => { + const name = after.table.name + const old = new Map(before.columns.map((c) => [c.name, c])) + const kept = new Set(after.columns.map((c) => c.name)) + const added = after.columns.filter((c) => !old.has(c.name)) + const dropped = before.columns.filter((c) => !kept.has(c.name)) + const changed = after.columns.filter((c) => old.has(c.name) && !same(definition(old.get(c.name)!), definition(c))) + const needsRebuild = + changed.length > 0 || + !same(before.table.primaryKey, after.table.primaryKey) || + !same(byName(before.uniques), byName(after.uniques)) || + added.some((c) => !addable(c, after)) || + dropped.some((c) => !droppable(c.name, before)) + const dropHint = (column: string): Hint => ({ type: "confirm_data_loss", kind: "column", entity: `${name}.${column}` }) + if (needsRebuild) { + const rebuild: SqliteMigrationOp = { + op: "rebuild_table", + table: after.table, + columns: after.columns, + uniques: after.uniques, + copy: after.columns.filter((c) => old.has(c.name)).map((c) => c.name), + } + confirm(dropped.map((c) => dropHint(c.name)), rebuild) + return + } + for (const column of added) ops.push({ op: "add_column", column }) + for (const column of dropped) confirm(dropHint(column.name), { op: "drop_column", table: name, name: column.name }) +} + +const byName = (uniques: ReadonlyArray) => + [...uniques].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) From 785f558459204996fe86bd4b5b700bab3c4298a3 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:57:17 +0200 Subject: [PATCH 03/15] feat(sqlite): rebuild tables for changes ALTER TABLE cannot make --- src/kit/sqlite-generate.test.ts | 100 +++++++++++++++++++++++++++++-- src/sqlite/shape.test-support.ts | 8 +++ 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index 4140808..6748fab 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -9,7 +9,7 @@ import { afterEach, beforeEach, expect, it } from "vitest" import { generate, type GenerateOptions, type KitConfig } from "../kit" import { STATEMENT_BREAKPOINT } from "../migrate/source" import { renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaTable } from "../schema" -import { shapeAfter } from "../sqlite/shape.test-support" +import { rowsAfter, shapeAfter } from "../sqlite/shape.test-support" const src = resolve(import.meta.dirname, "..") let dir = "" @@ -33,13 +33,16 @@ const folders = () => readdirSync(join(dir, "migrations")).sort() const migrationSql = (folder: string) => readFileSync(join(dir, "migrations", folder, "migration.sql"), "utf8") /** Every statement of every migration, in folder order, split as Alchemy and drizzle split them. */ -const allStatements = () => - folders().flatMap((folder) => +const allStatements = (seed: ReadonlyArray = []) => + folders().flatMap((folder, i) => [...(i === 1 ? seed : []), ...statementsOf(folder)]) + +const statementsOf = (folder: string) => ( + migrationSql(folder) .split(STATEMENT_BREAKPOINT) .map((s) => s.trim()) - .filter((s) => s.replace(/^\s*--.*$/gm, "").trim().length > 0), - ) + .filter((s) => s.replace(/^\s*--.*$/gm, "").trim().length > 0) +) /** The schema module's tables, created fresh. */ const freshShape = async (body: string) => { @@ -81,3 +84,90 @@ it("adds a nullable column, and a NOT NULL one with a literal default, with ALTE ]) await expectMigratedToEqualFresh(v2) }) + +const seedLinks = [`INSERT INTO "links" ("id", "url") VALUES ('a', 'https://a'), ('b', 'https://b')`] + +it("drops a column only once the data loss is confirmed, and keeps the other columns' rows", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text)`)) + const v2 = links(`id: SQLite.text, url: SQLite.text`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + expect(folders()).toHaveLength(1) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.note" }] }) + expect(second.plan).toEqual([{ label: "destructive", sql: ['ALTER TABLE "links" DROP COLUMN "note"'] }]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT * FROM "links" ORDER BY "id"')).toEqual([ + { id: "a", url: "https://a" }, + { id: "b", url: "https://b" }, + ]) +}) + +const rebuilds: ReadonlyArray = [ + ["a column type", `id: SQLite.text, url: SQLite.text, hits: SQLite.nullable(SQLite.text)`, `id: SQLite.text, url: SQLite.text, hits: SQLite.nullable(SQLite.integer)`], + ["nullability", `id: SQLite.text, url: SQLite.nullable(SQLite.text)`, `id: SQLite.text, url: SQLite.text`], + ["a default", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.column(SQLite.text, { default: "none" })`], + ["a UNIQUE column", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.column(SQLite.text, { unique: true })`], + ["a NOT NULL column with an expression default", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.text, at: SQLite.column(SQLite.text, { defaultExpr: "(CURRENT_TIMESTAMP)" })`], +] + +it.each(rebuilds)("rebuilds the table for %s, keeps its rows and recreates its indexes", async (_, before, after) => { + const index = `indexes: [SQLite.index("links_url", ["url"])],` + await generateFrom(links(before, index)) + const v2 = links(after, index) + const second = await generateFrom(v2) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild", "metadata"]) + expect(second.plan[0]!.sql).toEqual([ + "PRAGMA foreign_keys=OFF", + expect.stringMatching(/^CREATE TABLE "__new_links" \(/), + expect.stringMatching(/^INSERT INTO "__new_links"\("id", "url"(, "hits")?\) SELECT "id", "url"(, "hits")? FROM "links"$/), + 'DROP TABLE "links"', + 'ALTER TABLE "__new_links" RENAME TO "links"', + "PRAGMA foreign_keys=ON", + ]) + expect(second.plan[1]!.sql).toEqual(['CREATE INDEX IF NOT EXISTS "links_url" ON "links" ("url")']) + await expectMigratedToEqualFresh(v2) + const rows = await rowsAfter(allStatements(seedLinks), 'SELECT "id", "url" FROM "links" ORDER BY "id"') + expect(rows).toEqual([{ id: "a", url: "https://a" }, { id: "b", url: "https://b" }]) +}) + +it("rebuilds the table for a primary key change", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const v2 = ` +export const Links = SQLite.table("links", { + columns: { id: SQLite.text, url: SQLite.text }, + primaryKey: ["id", "url"], +})` + const second = await generateFrom(v2) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild"]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT count(*) AS n FROM "links"')).toEqual([{ n: 2 }]) +}) + +it("creates and drops indexes without touching the table", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, org: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + const v2 = links(`id: SQLite.text, url: SQLite.text, org: SQLite.text`, `indexes: [SQLite.uniqueIndex("links_org_url", ["org", "url"])],`) + const second = await generateFrom(v2) + expect(second.plan).toEqual([ + { label: "metadata", sql: ['DROP INDEX IF EXISTS "links_url"'] }, + { label: "metadata", sql: ['CREATE UNIQUE INDEX IF NOT EXISTS "links_org_url" ON "links" ("org", "url")'] }, + ]) + await expectMigratedToEqualFresh(v2) +}) + +it("asks to confirm a column a rebuild leaves behind", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text)`)) + const v2 = links(`id: SQLite.text, url: SQLite.integer`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints", details: [expect.stringContaining("links.note")] }) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.note" }] }) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild"]) + expect(second.plan[0]!.sql[2]).toBe('INSERT INTO "__new_links"("id", "url") SELECT "id", "url" FROM "links"') + await expectMigratedToEqualFresh(v2) +}) + +it("drops a table only once the data loss is confirmed", async () => { + await generateFrom(`${links(`id: SQLite.text, url: SQLite.text`)}\nexport const Old = SQLite.table("old", { columns: { id: SQLite.text }, indexes: [SQLite.index("old_id", ["id"])] })`) + const v2 = links(`id: SQLite.text, url: SQLite.text`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "table", entity: "old" }] }) + expect(second.plan).toEqual([{ label: "destructive", sql: ['DROP TABLE IF EXISTS "old"'] }]) + await expectMigratedToEqualFresh(v2) +}) diff --git a/src/sqlite/shape.test-support.ts b/src/sqlite/shape.test-support.ts index 72397da..12d5f69 100644 --- a/src/sqlite/shape.test-support.ts +++ b/src/sqlite/shape.test-support.ts @@ -25,3 +25,11 @@ export const shapeAfter = (statements: ReadonlyArray) => return shape }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + +/** The rows `select` returns after running `statements` on an empty database. */ +export const rowsAfter = (statements: ReadonlyArray, select: string) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: sqliteDialect }) + for (const sql of statements) yield* db.execute({ sql }) + return yield* db.query({ sql: select }) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) From 65e038b4b720d54c456ada3d2fca47bd044922de Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:57:59 +0200 Subject: [PATCH 04/15] feat(sqlite): rename tables and columns from confirm_rename hints --- src/kit/cli.ts | 4 ++- src/kit/sqlite-generate.test.ts | 24 ++++++++++++++ src/schema/diff.ts | 10 ++++-- src/schema/sqlite-diff.ts | 58 +++++++++++++++++++++++++++++++-- 4 files changed, 89 insertions(+), 7 deletions(-) diff --git a/src/kit/cli.ts b/src/kit/cli.ts index 9cbf026..2902643 100644 --- a/src/kit/cli.ts +++ b/src/kit/cli.ts @@ -68,7 +68,9 @@ const promptConfirm = (hint: Hint): Effect.Effect => const rl = createInterface({ input: process.stdin, output: process.stdout }) const answer = await rl.question( hint.type === "confirm_rename" - ? `Rename enum label ${hint.entity} to ${JSON.stringify(hint.to)}? Rows holding it will hold the new label. [y/N] ` + ? hint.kind === "enum_value" + ? `Rename enum label ${hint.entity} to ${JSON.stringify(hint.to)}? Rows holding it will hold the new label. [y/N] ` + : `Rename ${hint.kind} ${hint.entity} to ${hint.to}? [y/N] ` : `Drop ${hint.kind} ${hint.entity}? ${hint.kind === "enum" ? "Postgres refuses while a column uses it." : "Its data will be lost."} [y/N] `, ) rl.close() diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index 6748fab..05a8894 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -171,3 +171,27 @@ it("drops a table only once the data loss is confirmed", async () => { expect(second.plan).toEqual([{ label: "destructive", sql: ['DROP TABLE IF EXISTS "old"'] }]) await expectMigratedToEqualFresh(v2) }) + +it("renames a table and a column when a hint says so, keeping rows and indexes", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + const v2 = ` +export const Urls = SQLite.table("urls", { + columns: { id: SQLite.text, target: SQLite.text }, + primaryKey: ["id"], + indexes: [SQLite.index("links_url", ["target"])], +})` + // Unconfirmed, a rename is a drop plus a create. + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + const second = await generateFrom(v2, { + hints: [ + { type: "confirm_rename", kind: "table", entity: "links", to: "urls" }, + { type: "confirm_rename", kind: "column", entity: "urls.url", to: "target" }, + ], + }) + expect(second.plan).toEqual([ + { label: "metadata", sql: ['ALTER TABLE "links" RENAME TO "urls"'] }, + { label: "metadata", sql: ['ALTER TABLE "urls" RENAME COLUMN "url" TO "target"'] }, + ]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT "target" FROM "urls" ORDER BY "id"')).toEqual([{ target: "https://a" }, { target: "https://b" }]) +}) diff --git a/src/schema/diff.ts b/src/schema/diff.ts index 2abc6bd..a83b88e 100644 --- a/src/schema/diff.ts +++ b/src/schema/diff.ts @@ -33,11 +33,15 @@ export const Hint = Schema.Union([ /** `table`, `table.column`, or the enum's name. */ entity: Schema.String, }), - /** A Postgres enum label renamed in place (`ALTER TYPE ... RENAME VALUE`): rows holding it now hold `to`. */ + /** + * A rename. `enum_value`: a Postgres enum label renamed in place (`ALTER + * TYPE ... RENAME VALUE`), so rows holding it now hold `to`. `table` and + * `column` (SQLite): `ALTER TABLE ... RENAME`, instead of a drop and a create. + */ Schema.Struct({ type: Schema.Literal("confirm_rename"), - kind: Schema.Literal("enum_value"), - /** `.`. */ + kind: Schema.Literals(["enum_value", "table", "column"]), + /** `.`, the old table name, or `
.` with the table's new name. */ entity: Schema.String, to: Schema.String, }), diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts index 574cb83..c0b93eb 100644 --- a/src/schema/sqlite-diff.ts +++ b/src/schema/sqlite-diff.ts @@ -4,9 +4,10 @@ // dropping a table or a column needs a `confirm_data_loss` hint. import { canonicalJson, entityKey } from "./entities" -import { hintKey, type DiffResult, type Hint } from "./diff" +import { hintKey, type DiffResult, type Hint, type UnsupportedChange } from "./diff" import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" import type { SqliteMigrationOp } from "./sqlite-ops" +import { sqliteIdent } from "./sqlite-render" const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonicalJson(b) @@ -29,13 +30,16 @@ const opOrder: Record = { } export const diffSqliteSchemas = ( - prev: ReadonlyArray, + original: ReadonlyArray, next: ReadonlyArray, hints: ReadonlyArray = [], ): DiffResult => { const given = new Set(hints.map(hintKey)) const ops: Array = [] const missingHints: Array = [] + const unsupported: Array = [] + // Renames run first; everything after compares the renamed schema with the next one. + const prev = applyRenames(original, next, hints, ops, unsupported) /** Push `op` once every hint it needs was given; otherwise report the missing ones. */ const confirm = (needed: Hint | ReadonlyArray, op: SqliteMigrationOp): void => { const missing = (Array.isArray(needed) ? needed : [needed]).filter((hint) => !given.has(hintKey(hint))) @@ -90,7 +94,7 @@ export const diffSqliteSchemas = ( .sort(([a, i], [b, j]) => opOrder[a.op] - opOrder[b.op] || i - j) .map(([op]) => op), missingHints, - unsupported: [], + unsupported, } } @@ -159,3 +163,51 @@ const diffTable = ( const byName = (uniques: ReadonlyArray) => [...uniques].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + +/** + * Apply the `confirm_rename` hints for tables and columns: an op for each, and + * `prev` as it reads after them. SQLite's RENAME rewrites the indexes and + * constraints that name the table or column, so they are renamed here too. + * A hint that does not match the two schemas is reported. + */ +const applyRenames = ( + prev: ReadonlyArray, + next: ReadonlyArray, + hints: ReadonlyArray, + ops: Array, + unsupported: Array, +): ReadonlyArray => { + let entities = prev + const has = (list: ReadonlyArray, kind: "table" | "column", table: string, name?: string) => + list.some((e) => (kind === "table" ? e.kind === "table" && e.name === table : e.kind === "column" && e.table === table && e.name === name)) + for (const hint of hints) { + if (hint.type !== "confirm_rename" || hint.kind !== "table") continue + if (!has(entities, "table", hint.entity) || has(entities, "table", hint.to) || !has(next, "table", hint.to) || has(next, "table", hint.entity)) { + unsupported.push({ entity: hint.entity, message: `cannot rename to ${hint.to}: the old table must exist only before, and the new one only after` }) + continue + } + ops.push({ op: "rename_table", from: hint.entity, to: hint.to }) + entities = entities.map((e) => (e.kind === "table" ? (e.name === hint.entity ? { ...e, name: hint.to } : e) : e.table === hint.entity ? { ...e, table: hint.to } : e)) + } + for (const hint of hints) { + if (hint.type !== "confirm_rename" || hint.kind !== "column") continue + const dot = hint.entity.lastIndexOf(".") + const table = hint.entity.slice(0, dot) + const from = hint.entity.slice(dot + 1) + if (dot <= 0 || !has(entities, "column", table, from) || has(entities, "column", table, hint.to) || !has(next, "column", table, hint.to) || has(next, "column", table, from)) { + unsupported.push({ entity: hint.entity, message: `cannot rename to ${hint.to}: the old column must exist only before, and the new one only after` }) + continue + } + ops.push({ op: "rename_column", table, from, to: hint.to }) + const rename = (name: string) => (name === from ? hint.to : name) + const quoted = sqliteIdent(from) + entities = entities.map((e): SqliteSchemaEntity => { + if (e.kind === "table") return e.name === table && e.primaryKey !== null ? { ...e, primaryKey: { columns: e.primaryKey.columns.map(rename) } } : e + if (e.table !== table) return e + if (e.kind === "column") return { ...e, name: rename(e.name) } + if (e.kind === "unique") return { ...e, columns: e.columns.map(rename) } + return { ...e, columns: e.columns.map((part) => part.split(quoted).join(sqliteIdent(hint.to))) } + }) + } + return entities +} From 573ccbf2e956bbcc21cfe3d7a31e0a7c53193fa7 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:59:33 +0200 Subject: [PATCH 05/15] feat(sqlite): adopt a drizzle-kit 1.0 folder with generate --baseline --from-drizzle --- .../20261025075810_init/migration.sql | 34 ++ .../20261025075810_init/snapshot.json | 333 ++++++++++++++++++ .../__fixtures__/drizzle-sqlite/schema.ts.txt | 19 + src/kit/generate.ts | 6 +- src/kit/sqlite-generate.test.ts | 56 ++- src/migrate/source.ts | 6 +- src/schema/sqlite-drizzle.test.ts | 29 ++ src/schema/sqlite-drizzle.ts | 101 ++++++ 8 files changed, 579 insertions(+), 5 deletions(-) create mode 100644 src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql create mode 100644 src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json create mode 100644 src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt create mode 100644 src/schema/sqlite-drizzle.test.ts create mode 100644 src/schema/sqlite-drizzle.ts diff --git a/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql new file mode 100644 index 0000000..b105d8e --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql @@ -0,0 +1,34 @@ +CREATE TABLE `membership` ( + `org_id` text NOT NULL, + `user_id` text NOT NULL, + `role` text DEFAULT 'member' NOT NULL, + `weight` real DEFAULT 1.5 NOT NULL, + `active` integer DEFAULT true NOT NULL, + `avatar` blob, + `amount` numeric, + `email` text UNIQUE, + `created_at` text DEFAULT (CURRENT_TIMESTAMP) NOT NULL, + CONSTRAINT `membership_pk` PRIMARY KEY(`org_id`, `user_id`) +); +--> statement-breakpoint +CREATE TABLE `short_link` ( + `id` text PRIMARY KEY, + `code` text NOT NULL, + `target_url` text NOT NULL, + `target_hash` text NOT NULL, + `org_id` text, + `created_by` text, + `subject_type` text NOT NULL, + `subject_id` text NOT NULL, + `purpose` text NOT NULL, + `prospect_id` text, + `click_count` integer NOT NULL, + `first_accessed_at` text, + `last_accessed_at` text, + `created_at` text NOT NULL, + `updated_at` text NOT NULL +); +--> statement-breakpoint +CREATE INDEX `membership_user_role` ON `membership` (`user_id`,`role`);--> statement-breakpoint +CREATE UNIQUE INDEX `short_link_code` ON `short_link` (`code`);--> statement-breakpoint +CREATE INDEX `short_link_org` ON `short_link` (`org_id`); \ No newline at end of file diff --git a/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json new file mode 100644 index 0000000..1877350 --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json @@ -0,0 +1,333 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "815592fa-041d-4bbb-b8c4-a994b30babe5", + "prevIds": [ + "00000000-0000-0000-0000-000000000000" + ], + "ddl": [ + { + "name": "membership", + "entityType": "tables" + }, + { + "name": "short_link", + "entityType": "tables" + }, + { + "table": "membership", + "name": "org_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "user_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "role", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'member'", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "weight", + "type": "real", + "notNull": true, + "autoincrement": false, + "default": "1.5", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "active", + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "avatar", + "type": "blob", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "amount", + "type": "numeric", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "email", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "created_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "(CURRENT_TIMESTAMP)", + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "code", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "target_url", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "target_hash", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "org_id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "created_by", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "subject_type", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "subject_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "purpose", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "prospect_id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "click_count", + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "first_accessed_at", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "last_accessed_at", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "created_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "updated_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "membership_user_role", + "columns": [ + { + "value": "user_id", + "isExpression": false + }, + { + "value": "role", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "short_link", + "name": "short_link_code", + "columns": [ + { + "value": "code", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "short_link", + "name": "short_link_org", + "columns": [ + { + "value": "org_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "membership", + "name": "membership_pk", + "columns": [ + "org_id", + "user_id" + ], + "nameExplicit": false, + "entityType": "pks" + }, + { + "table": "short_link", + "name": "short_link_pk", + "columns": [ + "id" + ], + "nameExplicit": false, + "entityType": "pks" + }, + { + "table": "membership", + "name": "membership_email_unique", + "columns": [ + "email" + ], + "nameExplicit": false, + "entityType": "uniques" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt b/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt new file mode 100644 index 0000000..e886b25 --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt @@ -0,0 +1,19 @@ +// The drizzle-orm 1.0.0-rc.5 schema drizzle-kit 1.0.0-rc.5 `generate --name init` read to write this folder. +import { sqliteTable, text, integer, real, blob, numeric, index, uniqueIndex, primaryKey } from "drizzle-orm/sqlite-core" +import { sql } from "drizzle-orm" +export const shortLink = sqliteTable("short_link", { + id: text("id").primaryKey().notNull(), code: text("code").notNull(), targetUrl: text("target_url").notNull(), + targetHash: text("target_hash").notNull(), orgId: text("org_id"), createdBy: text("created_by"), + subjectType: text("subject_type").notNull(), subjectId: text("subject_id").notNull(), purpose: text("purpose").notNull(), + prospectId: text("prospect_id"), clickCount: integer("click_count").notNull(), + firstAccessedAt: text("first_accessed_at"), lastAccessedAt: text("last_accessed_at"), + createdAt: text("created_at").notNull(), updatedAt: text("updated_at").notNull(), +}, (t) => [uniqueIndex("short_link_code").on(t.code), index("short_link_org").on(t.orgId)]) +export const membership = sqliteTable("membership", { + orgId: text("org_id").notNull(), userId: text("user_id").notNull(), + role: text("role").notNull().default("member"), weight: real("weight").notNull().default(1.5), + active: integer("active", { mode: "boolean" }).notNull().default(true), + avatar: blob("avatar"), amount: numeric("amount"), + email: text("email").unique(), + createdAt: text("created_at").notNull().default(sql`(CURRENT_TIMESTAMP)`), +}, (t) => [primaryKey({ columns: [t.orgId, t.userId] }), index("membership_user_role").on(t.userId, t.role)]) diff --git a/src/kit/generate.ts b/src/kit/generate.ts index bfed7ea..c1d6f8e 100644 --- a/src/kit/generate.ts +++ b/src/kit/generate.ts @@ -10,6 +10,7 @@ import { fromRecord, isStagingName, STATEMENT_BREAKPOINT, type LoadedMigration, import type { MigrationDriver } from "../migrate/driver" import { diffSchemas, type DiffResult, type Hint } from "../schema/diff" import { fromDrizzleSnapshot } from "../schema/drizzle" +import { fromDrizzleSqliteSnapshot } from "../schema/sqlite-drizzle" import { ORIGIN_ID, type AnySchemaEntity, type SchemaDialect, type SchemaEntity } from "../schema/entities" import type { SchemaDefinitionError } from "../schema/problems" import { labelOf, renderOp, type MigrationFile, type MigrationOp } from "../schema/ops" @@ -203,10 +204,11 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption let snapshotEntities: ReadonlyArray let prevIds = analysis.baseIds if (options.baseline === "drizzle") { - if (dialect !== "postgres") return yield* new KitError({ code: "config", message: "--from-drizzle needs dialect: \"postgres\"" }) + if (dialect === "clickhouse") return yield* new KitError({ code: "config", message: "--from-drizzle needs dialect: \"postgres\" or \"sqlite\"" }) const source = [...analysis.legacy].reverse().find((m) => m.foreignSnapshot?.tool === "drizzle-kit") if (source === undefined) return yield* new KitError({ code: "config", message: "no drizzle-kit snapshot.json in the folder to start from" }) - const imported = fromDrizzleSnapshot(source.foreignSnapshot!.json) + const imported = + dialect === "sqlite" ? fromDrizzleSqliteSnapshot(source.foreignSnapshot!.json) : fromDrizzleSnapshot(source.foreignSnapshot!.json) if (imported.unsupported.length > 0) { return yield* new KitError({ code: "unsupported", diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index 05a8894..a6afd8b 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -1,7 +1,7 @@ // `generate` for SQLite, checked on real SQLite: the migrations it writes, // applied in order, must give the same PRAGMA results as creating the final // schema fresh. -import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { cpSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join, resolve } from "node:path" import { Effect } from "effect" @@ -29,7 +29,11 @@ const generateFrom = (body: string, options: GenerateOptions = {}) => { return Effect.runPromise(generate(config, dir, { name: `v${version}`, ...options })) } -const folders = () => readdirSync(join(dir, "migrations")).sort() +const folders = () => + readdirSync(join(dir, "migrations"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() const migrationSql = (folder: string) => readFileSync(join(dir, "migrations", folder, "migration.sql"), "utf8") /** Every statement of every migration, in folder order, split as Alchemy and drizzle split them. */ @@ -195,3 +199,51 @@ export const Urls = SQLite.table("urls", { await expectMigratedToEqualFresh(v2) expect(await rowsAfter(allStatements(seedLinks), 'SELECT "target" FROM "urls" ORDER BY "id"')).toEqual([{ target: "https://a" }, { target: "https://b" }]) }) + +// The tables of __fixtures__/drizzle-sqlite/schema.ts.txt, declared with effect-orm. +const drizzleEquivalent = ` +const named = [0]>(type: T, name: string) => SQLite.column(type, { name }) +const optionalText = (name: string) => named(SQLite.nullable(SQLite.text), name) +export const ShortLink = SQLite.table("short_link", { + columns: { + id: SQLite.text, code: SQLite.text, targetUrl: named(SQLite.text, "target_url"), targetHash: named(SQLite.text, "target_hash"), + orgId: optionalText("org_id"), createdBy: optionalText("created_by"), subjectType: named(SQLite.text, "subject_type"), + subjectId: named(SQLite.text, "subject_id"), purpose: SQLite.text, prospectId: optionalText("prospect_id"), + clickCount: named(SQLite.integer, "click_count"), firstAccessedAt: optionalText("first_accessed_at"), + lastAccessedAt: optionalText("last_accessed_at"), createdAt: named(SQLite.text, "created_at"), updatedAt: named(SQLite.text, "updated_at"), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], +}) +export const Membership = SQLite.table("membership", { + columns: { + orgId: named(SQLite.text, "org_id"), userId: named(SQLite.text, "user_id"), + role: SQLite.column(SQLite.text, { default: "member" }), weight: SQLite.column(SQLite.real, { default: 1.5 }), + active: SQLite.column(SQLite.boolean, { default: true }), avatar: SQLite.nullable(SQLite.blob), + amount: SQLite.nullable(SQLite.custom("numeric", SQLite.real.schema)), + email: SQLite.column(SQLite.nullable(SQLite.text), { unique: true }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["orgId", "userId"], + indexes: [SQLite.index("membership_user_role", ["userId", "role"])], +})` + +it("adopts a drizzle-kit 1.0 folder: a baseline from its snapshot, then no change for the same tables", async () => { + cpSync(join(import.meta.dirname, "__fixtures__", "drizzle-sqlite"), join(dir, "migrations"), { recursive: true }) + await expect(generateFrom(drizzleEquivalent)).rejects.toMatchObject({ code: "config", message: expect.stringContaining("--baseline") }) + const baseline = await generateFrom(drizzleEquivalent, { baseline: "drizzle" }) + expect(baseline.plan).toEqual([]) + const [, adopted] = folders() + expect(migrationSql(adopted!)).toMatch(/^-- Baseline/) + expect(JSON.parse(readFileSync(join(dir, "migrations", adopted!, "snapshot.json"), "utf8")).dialect).toBe("sqlite") + expect(await generateFrom(drizzleEquivalent)).toEqual({ written: undefined, plan: [] }) + // The comparison is live: one difference from the drizzle tables is a change. + const changed = await generateFrom(drizzleEquivalent.replace('"short_link_org", ["orgId"]', '"short_link_org", ["orgId", "purpose"]')) + expect(changed.plan.flatMap((p) => p.sql)).toEqual([ + 'DROP INDEX IF EXISTS "short_link_org"', + 'CREATE INDEX IF NOT EXISTS "short_link_org" ON "short_link" ("org_id", "purpose")', + ]) + // The folder drizzle-kit wrote, applied as it is, holds the same tables apart from the NOT NULL + // drizzle-kit drops from a text primary key. + expect(Object.keys(await shapeAfter(allStatements()))).toEqual(["membership", "short_link"]) +}) diff --git a/src/migrate/source.ts b/src/migrate/source.ts index 16cec6b..63897ae 100644 --- a/src/migrate/source.ts +++ b/src/migrate/source.ts @@ -86,6 +86,10 @@ const foreignSnapshotOf = (text: string): LoadedMigration["foreignSnapshot"] => : undefined } +/** The dialect a drizzle-kit snapshot names: `sqlite` (or `turso`) is SQLite; anything else is read as Postgres, as before. */ +const drizzleDialect = (json: unknown): SchemaDialect => + typeof json === "object" && json !== null && "dialect" in json && (json.dialect === "sqlite" || json.dialect === "turso") ? "sqlite" : "postgres" + /** * Each migration's parent migrations, from its snapshot's `prevIds`. A custom * migration copies its parent's entities, so several migrations can share one @@ -212,7 +216,7 @@ export const dialectOf = (migrations: ReadonlyArray): SchemaDia for (const m of migrations) { if (m.snapshot !== undefined) return m.snapshot.dialect if (m.file !== undefined) return "dialect" in m.file ? "postgres" : "clickhouse" - if (m.foreignSnapshot !== undefined) return "postgres" + if (m.foreignSnapshot !== undefined) return drizzleDialect(m.foreignSnapshot.json) } return undefined } diff --git a/src/schema/sqlite-drizzle.test.ts b/src/schema/sqlite-drizzle.test.ts new file mode 100644 index 0000000..5dabaa3 --- /dev/null +++ b/src/schema/sqlite-drizzle.test.ts @@ -0,0 +1,29 @@ +import { readdirSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { expect, it } from "vitest" +import { fromDrizzleSqliteSnapshot } from "./sqlite-drizzle" + +const fixture = join(import.meta.dirname, "..", "kit", "__fixtures__", "drizzle-sqlite") +const snapshot = () => { + const [folder] = readdirSync(fixture).filter((name) => /^\d{14}_/.test(name)) + return JSON.parse(readFileSync(join(fixture, folder!, "snapshot.json"), "utf8")) as { ddl: Array> } +} + +it("reads a drizzle-kit 1.0 primary key column as NOT NULL, and keeps UNIQUE constraints and index keys", () => { + const { entities, unsupported } = fromDrizzleSqliteSnapshot(snapshot()) + expect(unsupported).toEqual([]) + expect(entities).toContainEqual(expect.objectContaining({ kind: "column", table: "short_link", name: "id", notNull: true })) + expect(entities).toContainEqual({ kind: "unique", table: "membership", name: "membership_email_unique", columns: ["email"] }) + expect(entities).toContainEqual({ kind: "table", name: "membership", primaryKey: { columns: ["org_id", "user_id"] } }) + expect(entities).toContainEqual({ kind: "index", table: "membership", name: "membership_user_role", unique: false, columns: ['"user_id"', '"role"'] }) +}) + +it("reports what the entity model cannot hold", () => { + const json = snapshot() + json.ddl.push( + { entityType: "fks", table: "membership", name: "membership_org_fk" }, + { entityType: "columns", table: "membership", name: "n", type: "integer", notNull: true, autoincrement: true, default: null, generated: null }, + ) + expect(fromDrizzleSqliteSnapshot(json).unsupported).toEqual(["fks membership.membership_org_fk", "columns membership.n: AUTOINCREMENT"]) + expect(fromDrizzleSqliteSnapshot({ ...json, dialect: "postgresql" }).unsupported).toContain("dialect postgresql: only SQLite snapshots convert here") +}) diff --git a/src/schema/sqlite-drizzle.ts b/src/schema/sqlite-drizzle.ts new file mode 100644 index 0000000..8544277 --- /dev/null +++ b/src/schema/sqlite-drizzle.ts @@ -0,0 +1,101 @@ +// Reading a drizzle-kit 1.0 SQLite snapshot (`snapshot.json`, version 7) as +// entities, so `generate --baseline --from-drizzle` can adopt a drizzle-kit +// folder the way it does for Postgres. +// +// Only what the entity model holds converts: tables, columns, primary keys, +// indexes and UNIQUE constraints. Anything else (foreign keys, checks, views, +// generated or AUTOINCREMENT columns, partial indexes) is reported, and nothing +// is written. +// +// One reading is deliberate. drizzle-kit 1.0 treats a primary key column as +// not null and leaves `notNull: false` (and the NOT NULL keyword) out of its +// output. It is read as not null here, which is what effect-orm declares and +// writes for a key column. A table drizzle-kit created then still accepts a +// NULL in a single text key column until the table is rebuilt. + +import { sortEntities } from "./entities" +import { canonicalSqliteType, type SqliteSchemaEntity, type SqliteTableEntity } from "./sqlite-entities" +import { sqliteIdent } from "./sqlite-render" + +export interface DrizzleSqliteImport { + readonly entities: ReadonlyArray + /** Objects the snapshot holds that have no entity here. Empty when the import is complete. */ + readonly unsupported: ReadonlyArray +} + +type Json = Record + +const isRecord = (value: unknown): value is Json => typeof value === "object" && value !== null && !Array.isArray(value) +const str = (value: unknown): string => (typeof value === "string" ? value : String(value)) +const strings = (value: unknown): ReadonlyArray => (Array.isArray(value) ? value.map(str) : []) + +const indexPart = (part: unknown): string => { + if (typeof part === "string") return sqliteIdent(part) + if (!isRecord(part)) return str(part) + return part.isExpression === true ? str(part.value) : sqliteIdent(str(part.value)) +} + +/** Convert a drizzle-kit 1.0 SQLite snapshot. */ +export const fromDrizzleSqliteSnapshot = (json: unknown): DrizzleSqliteImport => { + const unsupported: Array = [] + if (!isRecord(json) || !Array.isArray(json.ddl)) { + return { entities: [], unsupported: ["not a drizzle-kit snapshot: it has no ddl list"] } + } + if (json.dialect !== "sqlite" && json.dialect !== "turso") { + unsupported.push(`dialect ${str(json.dialect)}: only SQLite snapshots convert here`) + } + const tables = new Map() + const entities: Array = [] + const positions = new Map() + for (const raw of json.ddl as ReadonlyArray) { + if (!isRecord(raw)) continue + const type = str(raw.entityType) + const where = `${type} ${raw.table !== undefined ? `${str(raw.table)}.` : ""}${str(raw.name)}` + switch (type) { + case "tables": + tables.set(str(raw.name), { kind: "table", name: str(raw.name), primaryKey: null }) + break + case "columns": { + const table = str(raw.table) + if (raw.generated !== null && raw.generated !== undefined) unsupported.push(`${where}: a generated column`) + if (raw.autoincrement === true) unsupported.push(`${where}: AUTOINCREMENT`) + const position = positions.get(table) ?? 0 + positions.set(table, position + 1) + entities.push({ + kind: "column", + table, + name: str(raw.name), + position, + type: canonicalSqliteType(str(raw.type)), + notNull: raw.notNull === true, + default: raw.default === null || raw.default === undefined ? null : str(raw.default), + }) + break + } + case "pks": { + const table = tables.get(str(raw.table)) + if (table === undefined) unsupported.push(`${where}: its table is not in the snapshot`) + else table.primaryKey = { columns: strings(raw.columns) } + break + } + case "indexes": + if (typeof raw.where === "string" && raw.where.length > 0) unsupported.push(`${where}: a partial index`) + entities.push({ + kind: "index", + table: str(raw.table), + name: str(raw.name), + unique: raw.isUnique === true, + columns: Array.isArray(raw.columns) ? raw.columns.map(indexPart) : [], + }) + break + case "uniques": + entities.push({ kind: "unique", table: str(raw.table), name: str(raw.name), columns: strings(raw.columns) }) + break + default: + unsupported.push(where) + } + } + const keyColumns = new Set([...tables.values()].flatMap((t) => (t.primaryKey?.columns ?? []).map((c) => `${t.name}.${c}`))) + const columns = entities.map((e) => (e.kind === "column" && keyColumns.has(`${e.table}.${e.name}`) ? { ...e, notNull: true } : e)) + return { entities: sortEntities([...tables.values(), ...columns]), unsupported } +} From 27d1ef2cf74d359f0024ee3276fda9c472681a43 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:59:55 +0200 Subject: [PATCH 06/15] test(sqlite): apply generated migrations to a local D1 database --- src/kit/sqlite-generate.test.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index a6afd8b..c633876 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -5,6 +5,7 @@ import { cpSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } import { tmpdir } from "node:os" import { join, resolve } from "node:path" import { Effect } from "effect" +import { Miniflare } from "miniflare" import { afterEach, beforeEach, expect, it } from "vitest" import { generate, type GenerateOptions, type KitConfig } from "../kit" import { STATEMENT_BREAKPOINT } from "../migrate/source" @@ -247,3 +248,28 @@ it("adopts a drizzle-kit 1.0 folder: a baseline from its snapshot, then no chang // drizzle-kit drops from a text primary key. expect(Object.keys(await shapeAfter(allStatements()))).toEqual(["membership", "short_link"]) }) + +it("writes migrations a local D1 database accepts, rebuilds included", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + await generateFrom(links(`id: SQLite.text, url: SQLite.column(SQLite.text, { unique: true }), hits: SQLite.column(SQLite.integer, { default: 0 })`, `indexes: [SQLite.index("links_url", ["url"])],`)) + expect(folders()).toHaveLength(2) + const mf = new Miniflare({ modules: true, script: "export default { fetch() { return new Response('') } }", compatibilityDate: "2026-07-01", d1Databases: ["DB"] }) + try { + const db = await mf.getD1Database("DB") + const [first, second] = folders() + await db.batch(statementsOf(first!).map((sql) => db.prepare(sql))) + await db.batch(seedLinks.map((sql) => db.prepare(sql))) + // One atomic batch per migration, as a D1 migration runner sends it. + await db.batch(statementsOf(second!).map((sql) => db.prepare(sql))) + const rows = await db.prepare('SELECT "id", "url", "hits" FROM "links" ORDER BY "id"').all() + expect(rows.results).toEqual([{ id: "a", url: "https://a", hits: 0 }, { id: "b", url: "https://b", hits: 0 }]) + const indexes = await db.prepare(`SELECT name, origin FROM pragma_index_list('links') ORDER BY name`).all() + expect(indexes.results).toEqual([ + { name: "links_url", origin: "c" }, + { name: "sqlite_autoindex_links_1", origin: "pk" }, + { name: "sqlite_autoindex_links_2", origin: "u" }, + ]) + } finally { + await mf.dispose() + } +}) From 375797d680f49ee2fd63e5db97361fd75be7a69e Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 10:00:08 +0200 Subject: [PATCH 07/15] fix(migrate): refuse SQLite folders until the runner has a SQLite ledger --- src/kit/sqlite-generate.test.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index c633876..e395a3d 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -7,7 +7,8 @@ import { join, resolve } from "node:path" import { Effect } from "effect" import { Miniflare } from "miniflare" import { afterEach, beforeEach, expect, it } from "vitest" -import { generate, type GenerateOptions, type KitConfig } from "../kit" +import { generate, readMigrations, type GenerateOptions, type KitConfig } from "../kit" +import * as Migrate from "../migrate" import { STATEMENT_BREAKPOINT } from "../migrate/source" import { renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaTable } from "../schema" import { rowsAfter, shapeAfter } from "../sqlite/shape.test-support" @@ -273,3 +274,19 @@ it("writes migrations a local D1 database accepts, rebuilds included", async () await mf.dispose() } }) + +it("refuses to run, record or verify SQLite migrations with the effect-orm runner, which has no SQLite ledger yet", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const migrations = await Effect.runPromise(readMigrations(join(dir, "migrations"))) + const driver = Effect.provideService(Migrate.MigrationDriver, {} as Migrate.MigrationDriver["Service"]) + for (const attempt of [ + Migrate.run({ migrations }), + Migrate.status(migrations), + Migrate.verify(migrations), + Migrate.baseline({ migrations, upTo: migrations[0]!.name }), + ]) { + await expect(Effect.runPromise(driver(attempt as Effect.Effect))).rejects.toMatchObject({ + message: expect.stringContaining("SQLite migrations are not run by effect-orm yet"), + }) + } +}) From 498730b25401f85746513dbfe4d0f93aa33812b3 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 10:00:21 +0200 Subject: [PATCH 08/15] docs(kit): name SQLite in the CLI help --- src/kit/cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/cli.ts b/src/kit/cli.ts index 2902643..3b7cdec 100644 --- a/src/kit/cli.ts +++ b/src/kit/cli.ts @@ -9,7 +9,7 @@ import { Hints, type Hint } from "../schema/diff" import { check, generate, KitError, readMigrations, type KitConfig } from "./generate" import { pull } from "./pull" -const help = `effect-orm: schema migrations for ClickHouse and Postgres +const help = `effect-orm: schema migrations for ClickHouse and Postgres, and generated SQLite migrations generate [--name x] [--custom] Diff the schema against the migrations folder and write the next migration [--hints ] [--hints-file ] @@ -19,7 +19,7 @@ const help = `effect-orm: schema migrations for ClickHouse and Postgres Write table definitions from the database (ClickHouse). Default --out: the config's schema module, when it names one. Then run generate --baseline check Validate the migrations folder (snapshot chain, branch conflicts) - migrate [--strict] Apply pending migrations (needs config.driver) + migrate [--strict] Apply pending migrations (needs config.driver; not SQLite yet) status Applied, pending, partial, or changed, per migration verify Compare the database with the last applied snapshot baseline Record and every one before it as applied, without running them From cf52e2f0d85fec1bbf5c585c772f260652594c81 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 10:02:06 +0200 Subject: [PATCH 09/15] docs(sqlite): document generated SQLite migrations and drizzle-kit adoption --- .changeset/sqlite-generate.md | 10 +++++ README.md | 2 +- docs/README.md | 2 +- docs/migrations.md | 74 ++++++++++++++++++++++++++++++++++- docs/reference.md | 7 ++-- docs/sqlite.md | 7 +++- src/schema.ts | 11 ++++++ tests/package-consumer.mts | 3 ++ 8 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 .changeset/sqlite-generate.md diff --git a/.changeset/sqlite-generate.md b/.changeset/sqlite-generate.md new file mode 100644 index 0000000..0630148 --- /dev/null +++ b/.changeset/sqlite-generate.md @@ -0,0 +1,10 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`effect-orm generate` supports `dialect: "sqlite"`. It diffs `SQLite.table` definitions against +the migrations folder and writes `_/migration.sql` plus `snapshot.json`, the +drizzle-kit 1.0 layout that Alchemy's D1 resource applies. `ALTER TABLE` adds, drops and renames +where SQLite allows it; other column, primary key and UNIQUE changes rebuild the table. Renames +need a `confirm_rename` hint. `generate --baseline --from-drizzle` adopts a drizzle-kit 1.0 +SQLite folder. Snapshots gain a `sqlite` dialect. `/migrate` refuses SQLite folders for now. diff --git a/README.md b/README.md index aef1e51..0e9299e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproduc Formerly `@maple-dev/effect-clickhouse`. SQLite and Cloudflare D1 use the same builder and row decoder through the new -[`/sqlite` entry](./docs/sqlite.md), which also writes SQLite table DDL. SQLite migration generation is not implemented. +[`/sqlite` entry](./docs/sqlite.md), which also writes SQLite table DDL. `effect-orm generate` writes SQLite migrations; effect-orm does not apply them yet. [Read the documentation](https://effect-clickhouse.maple.dev) · [Getting started](./docs/getting-started.md) · [Recipes](./docs/recipes.md) diff --git a/docs/README.md b/docs/README.md index e9499a4..000b160 100644 --- a/docs/README.md +++ b/docs/README.md @@ -102,4 +102,4 @@ See [Benchmarking](./benchmarking.md) and the ## SQLite and D1 -[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, table DDL, Effect SQL drivers, tenant checks and transaction limits. SQLite migration generation is not implemented. +[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, table DDL, Effect SQL drivers, tenant checks and transaction limits. `effect-orm generate` writes SQLite migrations; effect-orm does not apply them yet. diff --git a/docs/migrations.md b/docs/migrations.md index 67f9e26..694230e 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,6 +1,6 @@ # Schema and migrations -Every table is declared with its DDL: `table` from `/clickhouse` or `/postgres` carries the +Every table is declared with its DDL: `table` from `/clickhouse`, `/postgres` or `/sqlite` carries the engine, keys, indexes, and defaults beside the columns the query builder reads. You can ignore that and manage the database elsewhere, or let three opt-in entry points turn the definitions into migrations: @@ -17,6 +17,7 @@ check and folder loading are shared; table definitions, the diff, the DDL, the r drift detection are per database, because the two differ where it matters: ClickHouse DDL is not transactional and cannot change most things in place, and Postgres DDL is and can. The sections below describe ClickHouse first; [Postgres](#postgres) covers what differs. +[SQLite](#sqlite) has `generate` only: another tool applies its migrations for now. ## Defining tables @@ -420,3 +421,74 @@ drizzle-kit recorded, so every place they disagree (a constraint name, a default op to accept or fix. `verify` against the baseline also finds objects the database has and drizzle-kit's snapshot does not, such as a table a hand-written migration created and nothing dropped. + +## SQLite + +Set `dialect: "sqlite"` and export `SQLite.table` values with their DDL from the schema module: + +```ts title="sqlite-kit-config.ts" +import { defineConfig } from "@maple-dev/effect-orm/kit" + +export default defineConfig({ dialect: "sqlite", schema: "./src/db/schema.ts", out: "./migrations" }) +``` + +`effect-orm generate` writes `_/migration.sql` and `snapshot.json`, the +drizzle-kit 1.0 layout. Statements are split by `--> statement-breakpoint`, each under a comment +naming its label. A SQLite migration is always SQL; there is no `migration.json` for it. Tools +that apply drizzle-kit folders apply it as it is, for example Alchemy's +`Cloudflare.D1.Database({ migrations: "./migrations" })`, which runs each folder's statements as one +D1 batch. `snapshot.json` is for `generate` and `check`; those tools ignore it. + +What each change becomes: + +| Change | SQL | +| --- | --- | +| New table, dropped table | `CREATE TABLE`, `DROP TABLE` (confirm the data loss) | +| New column | `ALTER TABLE ... ADD COLUMN`, when SQLite allows it: the column is nullable or has a literal default, and is not in the primary key or a UNIQUE constraint. Otherwise a rebuild | +| Dropped column | `ALTER TABLE ... DROP COLUMN` (confirm the data loss), unless it is in the primary key or a UNIQUE constraint. Otherwise a rebuild | +| A column's type, nullability or default; the primary key; a UNIQUE column | A table rebuild | +| New or changed index | `DROP INDEX`, `CREATE [UNIQUE] INDEX` | +| Renamed table or column | `ALTER TABLE ... RENAME`, only with a `confirm_rename` hint | + +A rebuild follows [SQLite's procedure](https://www.sqlite.org/lang_altertable.html#otheralter) +the way drizzle-kit writes it: `PRAGMA foreign_keys=OFF`, `CREATE TABLE "__new_
"` with the +new definition, `INSERT INTO "__new_
" (...) SELECT ...` for every column both tables +have, `DROP TABLE`, `ALTER TABLE "__new_
" RENAME TO "
"`, `PRAGMA foreign_keys=ON`, +then every index of the table again. A copy fails when the rows do not fit the new table, such +as a column made NOT NULL over existing NULLs or a new NOT NULL column with no default. A +column a rebuild leaves out needs the same `confirm_data_loss` hint as a dropped column. + +A rename is never guessed. Without a hint it reads as a drop plus an add, and the drop asks for +confirmation. Pass the rename instead: + +```json +[ + { "type": "confirm_rename", "kind": "table", "entity": "links", "to": "urls" }, + { "type": "confirm_rename", "kind": "column", "entity": "urls.url", "to": "target" } +] +``` + +`entity` is the old table name, or `
.` with the table's new name. Table +renames run first. Indexes keep their names and follow the rename. + +### Adopting a drizzle-kit SQLite folder + +`effect-orm generate --baseline --from-drizzle` reads drizzle-kit 1.0's last `snapshot.json` +(version 7, `dialect: "sqlite"`): tables, columns, primary keys, indexes and UNIQUE +constraints. Foreign keys, checks, views, generated and AUTOINCREMENT columns and partial +indexes are listed, and nothing is written. The baseline runs nothing. A later `generate` +compares your `SQLite.table` definitions with it; when they declare the same tables, it writes +nothing. The tests prove this with a folder drizzle-kit 1.0.0-rc.5 generated. + +drizzle-kit 1.0 leaves `NOT NULL` out of a primary key column (`"id" text PRIMARY KEY`), so +SQLite then accepts a NULL key in a `text` key column. Its snapshot marks the column +`notNull: false`, but drizzle-kit treats a key column as not null. The baseline reads it as not +null too, so a `SQLite.table` key column (always `NOT NULL`) matches it and no rebuild is +written. The database keeps accepting a NULL key until that table is rebuilt for another change. + +The drizzle-kit 0.x layout (`meta/_journal.json` and flat `.sql` files) is not read. Run +`drizzle-kit up` to convert it first. + +`/migrate` does not run SQLite folders yet: `run`, `status`, `verify` and `baseline` fail with a +`MigrateSourceError` for one. Keep applying them with the tool you use now. + diff --git a/docs/reference.md b/docs/reference.md index 4090ec5..d948955 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -510,7 +510,7 @@ placeholders, double-quoted identifiers. | `SqlQuery`, `compileQuery` | `/sql` | | `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | | `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | -| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint`; for SQLite, `sqliteEntitiesOf`, `renderSqliteSchema`, `renderSqliteCreateTable`, `renderSqliteCreateIndex` and the `SqliteTableEntity`, `SqliteColumnEntity`, `SqliteIndexEntity`, `SqliteUniqueEntity` schemas | `/schema` | +| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint`; for SQLite, `sqliteEntitiesOf`, `renderSqliteSchema`, `renderSqliteCreateTable`, `renderSqliteCreateIndex`, `diffSqliteSchemas`, `fromDrizzleSqliteSnapshot`, `SqliteMigrationOp`, `renderSqliteOp`, `renderSqliteMigrationSql`, `labelOfSqlite`, `rebuildName`, the `SqliteSnapshot` schema and the `SqliteTableEntity`, `SqliteColumnEntity`, `SqliteIndexEntity`, `SqliteUniqueEntity` schemas | `/schema` | | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | @@ -592,5 +592,6 @@ drizzle-kit 1.0 writes and names it. **Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `TableDdl`, `SqliteSchemaTable`, `DdlExpr`, `DdlKey`, `DefinitionProblem`. A definition that cannot render records a problem; `S.sqliteEntitiesOf` fails with a `SchemaDefinitionError`, and `S.renderSqliteSchema` writes -the `CREATE TABLE` and `CREATE INDEX` statements. There is no SQLite diff, snapshot or migration -runner yet. +the `CREATE TABLE` and `CREATE INDEX` statements. `effect-orm generate` with `dialect: "sqlite"` +writes migrations for them; see [SQLite migrations](./migrations.md#sqlite). The `/migrate` +runner does not apply SQLite migrations yet. diff --git a/docs/sqlite.md b/docs/sqlite.md index a03c56f..b6ce4d1 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -82,8 +82,11 @@ The tests apply this DDL and drizzle-kit's SQL for the same tables to real SQLit 1.0 drops `NOT NULL` from a primary key column, so SQLite then accepts a NULL key in a `text` key column. This entry keeps `NOT NULL`. -There is no SQLite schema diff, introspection, migration ledger or kit dialect yet. Run the -rendered statements yourself, or keep migrating with your current tool. `WITHOUT ROWID`, +`effect-orm generate` with `dialect: "sqlite"` diffs these tables against the migrations folder +and writes the next migration, in the drizzle-kit 1.0 folder layout. See +[SQLite migrations](./migrations.md#sqlite). effect-orm does not apply SQLite migrations yet, +and has no SQLite introspection or drift check: apply the folder with a tool that reads +drizzle-kit folders, such as Alchemy's D1 resource. `WITHOUT ROWID`, `STRICT`, `AUTOINCREMENT`, partial indexes, foreign keys and `CHECK` constraints are not declared yet. diff --git a/src/schema.ts b/src/schema.ts index 6d6910b..0afb383 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -23,6 +23,7 @@ export { SchemaEntity, Snapshot, SNAPSHOT_VERSION, + SqliteSnapshot, TableEntity, canonicalJson, entityKey, @@ -96,6 +97,16 @@ export { SqliteUniqueEntity, canonicalSqliteType, } from "./schema/sqlite-entities" +export { diffSqliteSchemas } from "./schema/sqlite-diff" +export { fromDrizzleSqliteSnapshot, type DrizzleSqliteImport } from "./schema/sqlite-drizzle" +export { + SqliteMigrationOp, + labelOfSqlite, + rebuildName, + renderSqliteMigrationSql, + renderSqliteOp, + type SqliteOpLabel, +} from "./schema/sqlite-ops" export { renderSqliteColumnDefinition, renderSqliteCreateIndex, diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index fc6fc79..1b68663 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -111,3 +111,6 @@ assert.deepEqual(await Effect.runPromise(Effect.map(S.sqliteEntitiesOf([sqliteLi 'CREATE TABLE IF NOT EXISTS "consumer_links" (\n\t"id" text PRIMARY KEY NOT NULL,\n\t"org_id" text\n)', 'CREATE INDEX IF NOT EXISTS "consumer_links_org" ON "consumer_links" ("org_id")', ]) +const sqliteNext = await Effect.runPromise(S.sqliteEntitiesOf([sqliteLinks])) +assert.deepEqual(S.diffSqliteSchemas(sqliteNext, sqliteNext).ops, []) +assert.equal((await Effect.runPromise(S.makeSnapshot(sqliteNext, [S.ORIGIN_ID], "sqlite"))).dialect, "sqlite") From b6e20675e6b7c2847ac91c0b5017abc2d74297ee Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 10:02:28 +0200 Subject: [PATCH 10/15] docs(sqlite): explain the SQLite diff in its header --- src/schema/sqlite-diff.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts index c0b93eb..8a5542a 100644 --- a/src/schema/sqlite-diff.ts +++ b/src/schema/sqlite-diff.ts @@ -1,7 +1,11 @@ // Diff two SQLite schemas into migration ops. // // Pure and offline, like `diffPgSchemas`, with the same data-loss rule: -// dropping a table or a column needs a `confirm_data_loss` hint. +// dropping a table or a column needs a `confirm_data_loss` hint. A change +// SQLite's ALTER TABLE can make becomes an ALTER TABLE; any other change to a +// table becomes one `rebuild_table` op that carries the whole new table (see +// `sqlite-ops.ts`). Renames are never guessed: a `confirm_rename` hint turns +// a drop plus an add into a RENAME. import { canonicalJson, entityKey } from "./entities" import { hintKey, type DiffResult, type Hint, type UnsupportedChange } from "./diff" @@ -14,9 +18,9 @@ const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonical const pick = (entities: ReadonlyArray, kind: K) => entities.filter((e): e is Extract => e.kind === kind) -// Indexes go first, so no dropped column is still indexed; new tables and -// columns exist before the indexes that need them; a rebuilt table's indexes -// are created after it. +// Renames first, so later ops use the new names. Index drops before column +// drops, so no dropped column is still indexed. Tables and columns exist +// before the indexes that need them, and a rebuilt table before its indexes. const opOrder: Record = { rename_table: 0, rename_column: 1, @@ -98,7 +102,6 @@ export const diffSqliteSchemas = ( } } - interface TableState { readonly table: SqliteTableEntity readonly columns: ReadonlyArray From efd8bb3d802504016bfd5c6d19cb35e14a528e00 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:10:54 +0100 Subject: [PATCH 11/15] fix(sqlite): migrations keep rows, never drop a table's last column Found by fuzzing migrations between random schemas on seeded rows: - A rebuild that keeps no column (every old column dropped, new ones added) wrote no INSERT, so it deleted every row though only the columns' data was confirmed lost. It now copies the rowids, and each row takes the new columns' defaults. - Replacing every column of a table dropped them before adding the new ones, and SQLite cannot drop a table's last column. Column adds now run before column drops. - A column made NOT NULL that has a default failed the rebuild over existing NULLs: INSERT ... SELECT passes an explicit NULL, which does not take the default. The copy is COALESCE(column, default) now. docs/migrations.md: a rebuild under D1 or any transaction fires ON DELETE CASCADE in tables that reference the rebuilt one (PRAGMA foreign_keys=OFF does nothing there), a rebuild renumbers rowids, and ADD COLUMN appends. Co-Authored-By: Claude Opus 5.5 --- docs/migrations.md | 25 ++++++++++++++++++++++--- src/kit/sqlite-generate.test.ts | 15 ++++++++++++++- src/schema/sqlite-diff.ts | 14 ++++++++------ src/schema/sqlite-ops.ts | 16 +++++++++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/migrations.md b/docs/migrations.md index 694230e..5e8f08d 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -454,9 +454,28 @@ A rebuild follows [SQLite's procedure](https://www.sqlite.org/lang_altertable.ht the way drizzle-kit writes it: `PRAGMA foreign_keys=OFF`, `CREATE TABLE "__new_
"` with the new definition, `INSERT INTO "__new_
" (...) SELECT ...` for every column both tables have, `DROP TABLE`, `ALTER TABLE "__new_
" RENAME TO "
"`, `PRAGMA foreign_keys=ON`, -then every index of the table again. A copy fails when the rows do not fit the new table, such -as a column made NOT NULL over existing NULLs or a new NOT NULL column with no default. A -column a rebuild leaves out needs the same `confirm_data_loss` hint as a dropped column. +then every index of the table again. A column made NOT NULL that has a default is copied as +`COALESCE(, )`, so existing NULLs take the default. A copy fails when the rows +do not fit the new table, such as a column made NOT NULL over existing NULLs with no default, or +a new UNIQUE constraint over duplicate values. A column a rebuild leaves out needs the same +`confirm_data_loss` hint as a dropped column. + +> [!WARNING] +> `PRAGMA foreign_keys=OFF` does nothing inside a transaction, and D1 and most migrators run each +> migration in one. There, the rebuild's `DROP TABLE` deletes every row of the old table first, so +> a table that references it with `ON DELETE CASCADE` loses its rows, and `ON DELETE SET NULL` +> clears them. effect-orm declares no foreign keys, but a database may have them from another +> tool. Before applying a rebuild of a referenced table, check for such keys +> (`SELECT * FROM pragma_foreign_key_list('')`) and, if there are any, run the rebuild +> outside a transaction with foreign keys off. + +A rebuild renumbers rowids: the copy may read the old rows in any order. A table whose rows are +referenced by rowid (rather than by a declared key) should declare an `integer` primary key, which +is the rowid and is copied. + +`ALTER TABLE ... ADD COLUMN` always appends, so a column declared between two existing ones is +last in the database and first-to-last order differs from a fresh create. Name columns in +`INSERT` and `SELECT`, as the builder does. A rename is never guessed. Without a hint it reads as a drop plus an add, and the drop asks for confirmation. Pass the rename instead: diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index e395a3d..84b6187 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -124,7 +124,8 @@ it.each(rebuilds)("rebuilds the table for %s, keeps its rows and recreates its i expect(second.plan[0]!.sql).toEqual([ "PRAGMA foreign_keys=OFF", expect.stringMatching(/^CREATE TABLE "__new_links" \(/), - expect.stringMatching(/^INSERT INTO "__new_links"\("id", "url"(, "hits")?\) SELECT "id", "url"(, "hits")? FROM "links"$/), + // A NOT NULL column with a default copies through COALESCE, so old NULLs take the default. + expect.stringMatching(/^INSERT INTO "__new_links"\("id", "url"(, "hits")?\) SELECT "id", ("url"|COALESCE\("url", 'none'\))(, "hits")? FROM "links"$/), 'DROP TABLE "links"', 'ALTER TABLE "__new_links" RENAME TO "links"', "PRAGMA foreign_keys=ON", @@ -135,6 +136,18 @@ it.each(rebuilds)("rebuilds the table for %s, keeps its rows and recreates its i expect(rows).toEqual([{ id: "a", url: "https://a" }, { id: "b", url: "https://b" }]) }) +it("fills existing NULLs with the default when a column becomes NOT NULL with one", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.nullable(SQLite.text)`)) + const v2 = links(`id: SQLite.text, url: SQLite.column(SQLite.text, { default: "none" })`) + await generateFrom(v2) + await expectMigratedToEqualFresh(v2) + const seed = [`INSERT INTO "links" ("id", "url") VALUES ('a', 'https://a'), ('b', NULL)`] + expect(await rowsAfter(allStatements(seed), 'SELECT "id", "url" FROM "links" ORDER BY "id"')).toEqual([ + { id: "a", url: "https://a" }, + { id: "b", url: "none" }, + ]) +}) + it("rebuilds the table for a primary key change", async () => { await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) const v2 = ` diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts index 8a5542a..aa82aaf 100644 --- a/src/schema/sqlite-diff.ts +++ b/src/schema/sqlite-diff.ts @@ -19,16 +19,18 @@ const pick = (entities: ReadonlyArray => e.kind === kind) // Renames first, so later ops use the new names. Index drops before column -// drops, so no dropped column is still indexed. Tables and columns exist -// before the indexes that need them, and a rebuilt table before its indexes. +// drops, so no dropped column is still indexed. Column adds before column +// drops: SQLite cannot drop a table's last column, and replacing every column +// would otherwise try to. Tables and columns exist before the indexes that +// need them, and a rebuilt table before its indexes. const opOrder: Record = { rename_table: 0, rename_column: 1, drop_index: 2, - drop_column: 3, - drop_table: 4, - create_table: 5, - add_column: 6, + add_column: 3, + drop_column: 4, + drop_table: 5, + create_table: 6, rebuild_table: 7, create_index: 8, } diff --git a/src/schema/sqlite-ops.ts b/src/schema/sqlite-ops.ts index 241f861..32edcd6 100644 --- a/src/schema/sqlite-ops.ts +++ b/src/schema/sqlite-ops.ts @@ -57,6 +57,16 @@ export const rebuildName = (table: string): string => `__new_${table}` const list = (names: ReadonlyArray): string => names.map(sqliteIdent).join(", ") +/** + * What a rebuild copies into a column. A column that gains NOT NULL keeps its + * default for existing NULLs: an explicit NULL in `INSERT ... SELECT` does not + * take the column's default, so the copy would fail on the old NULLs. + */ +const copied = (column: SqliteColumnEntity | undefined, name: string): string => + column?.notNull === true && column.default !== null && column.default.toUpperCase() !== "NULL" + ? `COALESCE(${sqliteIdent(name)}, ${column.default})` + : sqliteIdent(name) + /** The statements one op runs, in order. */ export const renderSqliteOp = (op: SqliteMigrationOp): ReadonlyArray => { switch (op.op) { @@ -78,7 +88,11 @@ export const renderSqliteOp = (op: SqliteMigrationOp): ReadonlyArray => return [ "PRAGMA foreign_keys=OFF", created, - ...(op.copy.length === 0 ? [] : [`INSERT INTO ${sqliteIdent(temporary)}(${list(op.copy)}) SELECT ${list(op.copy)} FROM ${sqliteIdent(op.table.name)}`]), + // With no column in common the rows still carry over, every column taking its default: + // only the dropped columns' data was confirmed lost, not the table's rows. + op.copy.length === 0 + ? `INSERT INTO ${sqliteIdent(temporary)}(rowid) SELECT rowid FROM ${sqliteIdent(op.table.name)}` + : `INSERT INTO ${sqliteIdent(temporary)}(${list(op.copy)}) SELECT ${op.copy.map((name) => copied(op.columns.find((c) => c.name === name), name)).join(", ")} FROM ${sqliteIdent(op.table.name)}`, `DROP TABLE ${sqliteIdent(op.table.name)}`, `ALTER TABLE ${sqliteIdent(temporary)} RENAME TO ${sqliteIdent(op.table.name)}`, "PRAGMA foreign_keys=ON", From 6cac1d21ec2317992b7e0e61261589f434fd0652 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:10:54 +0100 Subject: [PATCH 12/15] test(fuzz): migrations between random SQLite schemas reach the fresh schema and keep rows Two random schemas over a shared pool of table and column names: new, dropped and changed tables, columns, types, nullability, literal and expression defaults, keys, UNIQUE columns and indexes. The migration diffSqliteSchemas writes runs inside a transaction on rows that fit both schemas. The PRAGMA shape must equal a fresh create of the new schema (column order aside) and every row must survive, holding its values or the new column's default. Pins the three counterexamples fixed in the previous commit. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/README.md | 1 + tests/fuzz/migrations.fuzz.test.ts | 298 +++++++++++++++++++++++++++++ tests/fuzz/regressions.ts | 25 +++ 3 files changed, 324 insertions(+) create mode 100644 tests/fuzz/migrations.fuzz.test.ts diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 25a6b4f..3bb213d 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -20,6 +20,7 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | +| `migrations.fuzz.test.ts` | a SQLite migration between two random schemas, run on seeded rows, gives a fresh create's PRAGMA shape and keeps the rows | a fresh create, and the seeded rows | | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | ## Adding a dialect diff --git a/tests/fuzz/migrations.fuzz.test.ts b/tests/fuzz/migrations.fuzz.test.ts new file mode 100644 index 0000000..ac9c345 --- /dev/null +++ b/tests/fuzz/migrations.fuzz.test.ts @@ -0,0 +1,298 @@ +// SQLite migrations: any schema change, applied to a database with rows, gives +// the database a fresh create would, and keeps the rows. +// +// Two random schemas share a pool of table and column names, so any pair holds +// a mix of new, dropped and changed tables, columns, keys, UNIQUE constraints, +// defaults and indexes. The migration `diffSqliteSchemas` writes from one to the +// other runs on a database seeded with rows that fit both, inside a transaction +// as D1 and drizzle's migrator run it. Then: +// +// same shape PRAGMA table_info / index_list / index_xinfo match a fresh +// create of the new schema (column order aside: ADD COLUMN appends) +// rows kept every seeded row is still there (as a multiset: a rebuild +// renumbers rowids), each kept column holding its value, and a +// new or newly NOT NULL column its default +import { DatabaseSync } from "node:sqlite" +import fc from "fast-check" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as SQLite from "@maple-dev/effect-orm/sqlite" +import { diffSqliteSchemas, renderSqliteOp, renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaEntity } from "@maple-dev/effect-orm/schema" +import { FuzzFailure, property, skip, withContext } from "./harness" + +// Schema plans + +type ColumnType = "text" | "integer" | "real" | "blob" +type Default = { readonly k: "literal" } | { readonly k: "expr" } | { readonly k: "now" } +interface ColumnPlan { + readonly name: string + readonly type: ColumnType + readonly nullable: boolean + readonly default?: Default + readonly unique: boolean +} +interface TablePlan { + readonly name: string + readonly columns: ReadonlyArray + readonly primaryKey: ReadonlyArray + readonly indexes: ReadonlyArray<{ readonly columns: ReadonlyArray; readonly unique: boolean }> +} +type SchemaPlan = ReadonlyArray + +const TABLES = ["t0", "t1"] as const +const COLUMNS = ["c0", "c1", "c2", "c3", "c4"] as const + +const defaultFor = (type: ColumnType): fc.Arbitrary => + type === "blob" + ? fc.constant(undefined) + : fc.option(fc.constantFrom({ k: "literal" }, { k: "expr" }, ...(type === "text" ? [{ k: "now" } as const] : [])), { nil: undefined, freq: 2 }) + +const columnPlan = (name: string): fc.Arbitrary => + fc.constantFrom("text", "integer", "real", "blob").chain((type) => + fc.record({ + name: fc.constant(name), + type: fc.constant(type), + nullable: fc.boolean(), + default: defaultFor(type), + unique: fc.integer({ min: 0, max: 4 }).map((n) => n === 0), + }), + ) + +const tablePlan = (name: string): fc.Arbitrary => + fc + // Shuffled, so a column shared by two schemas may sit at a different position. + .shuffledSubarray([...COLUMNS], { minLength: 1 }) + .chain((names) => fc.tuple(...names.map(columnPlan))) + .chain((columns) => { + const keyable = columns.filter((c) => !c.nullable).map((c) => c.name) + const names = columns.map((c) => c.name) + return fc.record({ + name: fc.constant(name), + columns: fc.constant(columns), + primaryKey: keyable.length === 0 ? fc.constant([]) : fc.shuffledSubarray(keyable, { maxLength: Math.min(2, keyable.length) }), + indexes: fc.array(fc.record({ columns: fc.shuffledSubarray(names, { minLength: 1, maxLength: Math.min(2, names.length) }), unique: fc.boolean() }), { maxLength: 2 }), + }) + }) + +const schemaPlan: fc.Arbitrary = fc + .subarray([...TABLES], { minLength: 1 }) + .chain((names) => fc.tuple(...names.map(tablePlan))) + +// Plans as SQLite.table definitions + +const literalOf = (type: ColumnType): string | number => (type === "text" ? "d" : type === "integer" ? 7 : 1.5) +const exprOf = (type: ColumnType): string => (type === "text" ? "(lower('X'))" : "(1 + 1)") +/** What a default reads back as, or undefined where it is not predictable (the current time). */ +const defaultValue = (column: ColumnPlan): unknown => + column.default?.k === "literal" ? literalOf(column.type) : column.default?.k === "expr" ? (column.type === "text" ? "x" : 2) : undefined + +const typeOf = (column: ColumnPlan) => { + const base = { text: SQLite.text, integer: SQLite.integer, real: SQLite.real, blob: SQLite.blob }[column.type] as SQLite.SqliteType + return column.nullable ? SQLite.nullable(base) : base +} + +const tablesOf = (plan: SchemaPlan) => + plan.map((table) => + SQLite.table(table.name, { + columns: Object.fromEntries( + table.columns.map((column) => [ + column.name, + SQLite.column(typeOf(column), { + ...(column.default?.k === "literal" ? { default: literalOf(column.type) as never } : {}), + ...(column.default?.k === "expr" ? { defaultExpr: exprOf(column.type) } : {}), + ...(column.default?.k === "now" ? { defaultExpr: "(CURRENT_TIMESTAMP)" } : {}), + ...(column.unique ? { unique: true as const } : {}), + } as never), + ]), + ), + ...(table.primaryKey.length > 0 ? { primaryKey: table.primaryKey as never } : {}), + indexes: table.indexes.map((index, i) => (index.unique ? SQLite.uniqueIndex : SQLite.index)(`${table.name}_i${i}`, index.columns as never)), + }), + ) + +const entitiesOf = (plan: SchemaPlan): ReadonlyArray => { + const exit = Effect.runSyncExit(sqliteEntitiesOf(tablesOf(plan) as never)) + if (exit._tag === "Failure") throw new FuzzFailure("a generated schema was refused", { plan, cause: String(exit.cause) }) + return exit.value +} + +// Rows + +/** A value for row `i` of a column: distinct per row, so UNIQUE and key constraints in either schema hold. */ +const seedValue = (column: ColumnPlan, i: number): string | number => + column.type === "integer" ? i + 1 : column.type === "real" ? i + 0.5 : `v${i}` + +const sqlValue = (value: unknown): string => + value === null ? "NULL" : typeof value === "number" ? String(value) : `'${String(value).replaceAll("'", "''")}'` + +/** + * Whether rows can exist in both schemas. A new NOT NULL column without a + * default has no value for old rows, and a UNIQUE constraint over new columns + * only gives every old row the same default, and a column that becomes the + * rowid (a one-column integer key) takes integers only; the migration then + * fails, as it should, so those pairs are checked with no rows. + */ +const rowsFit = (before: TablePlan, after: TablePlan): boolean => { + const old = new Set(before.columns.map((c) => c.name)) + const added = after.columns.filter((c) => !old.has(c.name)) + if (added.some((c) => !c.nullable && c.default === undefined)) return false + const allNew = (cols: ReadonlyArray) => cols.length > 0 && cols.every((c) => !old.has(c)) + // A one-column `integer` key is the rowid: SQLite refuses a value that is not an integer there. + const rowid = after.primaryKey.length === 1 ? after.columns.find((c) => c.name === after.primaryKey[0]) : undefined + const was = before.columns.find((c) => c.name === rowid?.name) + if (rowid?.type === "integer" && was !== undefined && was.type !== "integer") return false + return !( + allNew(after.primaryKey) || + after.columns.some((c) => c.unique && !old.has(c.name)) || + after.indexes.some((index) => index.unique && allNew(index.columns)) + ) +} + +interface Seed { + readonly table: TablePlan + readonly rows: ReadonlyArray> +} + +/** Rows for `before` that `after` accepts: NULL only where the new column allows it or fills it with a default. */ +const seedRows = (before: SchemaPlan, after: SchemaPlan, count: number, nulls: ReadonlyArray): ReadonlyArray => + before.map((table) => { + const next = after.find((t) => t.name === table.name) + const n = next === undefined || rowsFit(table, next) ? count : 0 + const keyed = new Set(table.primaryKey) + // NULLs filled with one default would collide under a UNIQUE constraint of the new table. + const uniqueLater = (name: string) => + next !== undefined && + (next.columns.some((c) => c.name === name && c.unique) || + next.primaryKey.includes(name) || + next.indexes.some((index) => index.unique && index.columns.includes(name))) + const rows = Array.from({ length: n }, (_, i) => + Object.fromEntries( + table.columns.map((column, j) => { + const later = next?.columns.find((c) => c.name === column.name) + const nullable = + column.nullable && + !keyed.has(column.name) && + (later === undefined || later.nullable || (later.default !== undefined && later.default.k !== "now" && !uniqueLater(column.name))) + return [column.name, nullable && nulls[(i * 7 + j) % nulls.length] ? null : seedValue(column, i)] + }), + ), + ) + return { table, rows } + }) + +const insertsOf = (seeds: ReadonlyArray) => + seeds.flatMap(({ table, rows }) => + rows.map((row) => `INSERT INTO "${table.name}" (${Object.keys(row).map((c) => `"${c}"`).join(", ")}) VALUES (${Object.values(row).map(sqlValue).join(", ")})`), + ) + +/** The rows a table should hold after the migration, by row; `undefined` cells are not predictable and not compared. */ +const expectedRows = (seed: Seed, after: TablePlan): ReadonlyArray> => + seed.rows.map((row) => + Object.fromEntries( + after.columns.map((column) => { + const fill = defaultValue(column) + if (!Object.hasOwn(row, column.name)) return [column.name, column.default === undefined ? null : fill] + const value = row[column.name] + // A column made NOT NULL with a default takes it for old NULLs (COALESCE in the copy). + if (value === null && !column.nullable && column.default !== undefined) return [column.name, fill] + return [column.name, value] + }), + ), + ) + +// Shape + +interface Shape { + readonly [table: string]: { readonly columns: ReadonlyArray; readonly indexes: ReadonlyArray } +} + +/** + * What SQLite reports, minus what legitimately depends on history: column + * order (ADD COLUMN appends, documented) and auto-index names and numbering. + */ +const shapeOf = (db: DatabaseSync): Shape => { + const all = (sql: string) => db.prepare(sql).all().map((row) => ({ ...row }) as Record) + const shape: Record = {} + for (const { name } of all("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")) { + const columns = all(`SELECT name, type, "notnull", dflt_value, pk FROM pragma_table_info('${name}')`).sort((a, b) => (a.name < b.name ? -1 : 1)) + const indexes = all(`SELECT name, "unique", origin, partial FROM pragma_index_list('${name}')`) + .map((index) => { + const keys = all(`SELECT seqno, name, "desc", coll FROM pragma_index_xinfo('${index.name}') WHERE "key" = 1`) + const label = String(index.name).startsWith("sqlite_autoindex_") ? `auto(${keys.map((k) => k.name).join(",")})` : index.name + return { ...index, name: label, keys } + }) + .sort((a, b) => (a.name < b.name ? -1 : 1)) + shape[name] = { columns, indexes } + } + return shape +} + +const run = (db: DatabaseSync, statements: ReadonlyArray) => { + for (const sql of statements) db.exec(sql) +} + +// The property + +const migrates = async ([before, after, count, nulls]: readonly [SchemaPlan, SchemaPlan, number, ReadonlyArray]) => { + const from = entitiesOf(before) + const to = entitiesOf(after) + // Every drop is confirmed: the property is about what a migration does, not the confirmation gate. + const hints = before.flatMap((table) => [ + { type: "confirm_data_loss" as const, kind: "table" as const, entity: table.name }, + ...table.columns.map((c) => ({ type: "confirm_data_loss" as const, kind: "column" as const, entity: `${table.name}.${c.name}` })), + ]) + const diff = diffSqliteSchemas(from, to, hints) + if (diff.missingHints.length > 0 || diff.unsupported.length > 0) { + throw new FuzzFailure("the diff needs more than every drop confirmed", { missingHints: diff.missingHints, unsupported: diff.unsupported }) + } + const migration = diff.ops.flatMap(renderSqliteOp) + const seeds = seedRows(before, after, count, nulls) + + const fresh = new DatabaseSync(":memory:") + const migrated = new DatabaseSync(":memory:") + try { + run(fresh, renderSqliteSchema(to)) + run(migrated, [...renderSqliteSchema(from), ...insertsOf(seeds)]) + await withContext({ before, after, migration, seeds }, () => { + try { + run(migrated, ["BEGIN", ...migration, "COMMIT"]) + } catch (error) { + throw new FuzzFailure(`the migration failed on rows that fit both schemas: ${(error as Error).message}`, {}) + } + expect(shapeOf(migrated)).toEqual(shapeOf(fresh)) + for (const seed of seeds) { + const next = after.find((t) => t.name === seed.table.name) + if (next === undefined || seed.rows.length === 0) continue + // A type change may convert a value (`'1'` to `1`); compared as text, it is the same value. + const text = (v: unknown) => (v === null || v === undefined ? null : String(v)) + const want = expectedRows(seed, next).map((row) => + Object.fromEntries(Object.entries(row).filter(([, v]) => v !== undefined).map(([k, v]) => [k, text(v)])), + ) + // Compared as a multiset: a rebuild's copy may read the old rows in index order and renumber + // their rowids, as SQLite's documented procedure (and drizzle-kit) does. + const keys = Object.keys(want[0] ?? {}) + const sorted = (rows: ReadonlyArray>) => rows.map((row) => JSON.stringify(keys.map((k) => row[k]))).sort() + const actual = migrated.prepare(`SELECT * FROM "${next.name}"`).all().map((row: Record) => Object.fromEntries(keys.map((k) => [k, text(row[k])]))) + expect(sorted(actual)).toEqual(sorted(want)) + } + }) + } finally { + fresh.close() + migrated.close() + } +} + +describe("SQLite migrations", () => { + const name = "migrations reach the fresh schema and keep the rows (sqlite)" + it( + name, + () => + property( + name, + fc.tuple(schemaPlan, schemaPlan, fc.integer({ min: 0, max: 3 }), fc.array(fc.boolean(), { minLength: 1, maxLength: 5 })), + (input) => (input[0].length === 0 ? skip("no tables") : migrates(input)), + { runs: 200 }, + ), + 300_000, + ) +}) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index c5618ba..ce3ded4 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -62,6 +62,31 @@ export const pinned: Record> = { "postgres:text[] select": [[]], "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + // SQLite migrations. + "migrations reach": [ + // sqlite-rebuild-no-shared-column: a rebuild keeping no column copied no rows. + [ + [{ name: "t0", columns: [{ name: "c2", type: "text", nullable: false, unique: true }], primaryKey: [], indexes: [] }], + [{ name: "t0", columns: [{ name: "c0", type: "blob", nullable: true, unique: false }], primaryKey: [], indexes: [] }], + 1, + [false], + ], + // sqlite-drop-last-column: replacing every column dropped the last one before adding. + [ + [{ name: "t0", columns: [{ name: "c1", type: "text", nullable: false, default: { k: "expr" }, unique: false }, { name: "c2", type: "text", nullable: false, unique: false }], primaryKey: [], indexes: [] }], + [{ name: "t0", columns: [{ name: "c3", type: "text", nullable: true, unique: false }], primaryKey: [], indexes: [] }], + 0, + [false], + ], + // A column made NOT NULL with a default over old NULLs (COALESCE in the copy). + [ + [{ name: "t0", columns: [{ name: "c0", type: "text", nullable: false, unique: false }, { name: "c1", type: "text", nullable: true, unique: false }], primaryKey: [], indexes: [] }], + [{ name: "t0", columns: [{ name: "c0", type: "text", nullable: false, unique: false }, { name: "c1", type: "text", nullable: false, default: { k: "literal" }, unique: false }], primaryKey: [], indexes: [] }], + 2, + [true], + ], + ], + // Hostile values. "hostile values are refused, never thrown (clickhouse)": [ { dialect: "clickhouse", column: "OrgId", site: "eq", value: 0n }, From a64293513c24ab27b52b004f67e258592b706c95 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:02:25 +0100 Subject: [PATCH 13/15] test(fuzz): migrations on D1 and with renames; drizzle-kit adoption round trip - The migration property runs on D1 too: each migration as one batch on a local workerd binding, as Alchemy's D1 resource applies it, compared with a fresh create on a second D1 database. - Inputs rename a shared table and one of its kept columns through confirm_rename hints (about half of all inputs do), so renames combine with every other change and with rebuilds. - A drizzle-kit 1.0 snapshot of any random schema (written as drizzle-kit writes it, key columns `notNull: false` included) imports with nothing left to generate. - Changesets no longer say SQLite DDL or migrations are not implemented. Co-Authored-By: Claude Opus 5.5 --- .changeset/quiet-sqlite-dialects.md | 2 +- .changeset/sqlite-table-ddl.md | 2 +- tests/fuzz/README.md | 2 +- tests/fuzz/migrations.fuzz.test.ts | 302 ++++++++++++++++++++++------ 4 files changed, 240 insertions(+), 68 deletions(-) diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md index 9f58d58..a01c803 100644 --- a/.changeset/quiet-sqlite-dialects.md +++ b/.changeset/quiet-sqlite-dialects.md @@ -7,4 +7,4 @@ Add the `/sqlite` query entry with SQLite codecs, functions and positional bindi and explicit refusal of interactive transactions. `timestamp` stores SQLite's own `YYYY-MM-DD HH:MM:SS.SSS` UTC text, so it compares correctly with what `strftime` writes; `LIKE` escapes with `\` as on every dialect; text with U+0000 and integers past 2^53 are refused rather -than silently changed. SQLite migration generation is not implemented yet. +than silently changed. diff --git a/.changeset/sqlite-table-ddl.md b/.changeset/sqlite-table-ddl.md index 629d3ec..2ae47f4 100644 --- a/.changeset/sqlite-table-ddl.md +++ b/.changeset/sqlite-table-ddl.md @@ -8,4 +8,4 @@ defaults (`SQLite.timestampNow` for a `timestamp` column), unique columns, and n unique indexes. `sqliteEntitiesOf` and `renderSqliteSchema` from `/schema` check a schema and write its `CREATE TABLE` and `CREATE INDEX` statements. `table(name, columns, options?)` and -`table(name, { external: true, columns })` stay query-only. SQLite migrations are not generated yet. +`table(name, { external: true, columns })` stay query-only. diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 3bb213d..8e59da0 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -20,7 +20,7 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | -| `migrations.fuzz.test.ts` | a SQLite migration between two random schemas, run on seeded rows, gives a fresh create's PRAGMA shape and keeps the rows | a fresh create, and the seeded rows | +| `migrations.fuzz.test.ts` | a SQLite migration between two random schemas (with table and column renames), run on seeded rows on node:sqlite and on D1 (one batch per migration), gives a fresh create's PRAGMA shape and keeps the rows; a drizzle-kit snapshot of any schema imports with nothing to generate | a fresh create, the seeded rows, and our own entities | | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | ## Adding a dialect diff --git a/tests/fuzz/migrations.fuzz.test.ts b/tests/fuzz/migrations.fuzz.test.ts index ac9c345..b3accd9 100644 --- a/tests/fuzz/migrations.fuzz.test.ts +++ b/tests/fuzz/migrations.fuzz.test.ts @@ -15,9 +15,10 @@ import { DatabaseSync } from "node:sqlite" import fc from "fast-check" import { Effect } from "effect" -import { describe, expect, it } from "vitest" +import { Miniflare } from "miniflare" +import { afterAll, afterEach, describe, expect, it } from "vitest" import * as SQLite from "@maple-dev/effect-orm/sqlite" -import { diffSqliteSchemas, renderSqliteOp, renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaEntity } from "@maple-dev/effect-orm/schema" +import { diffSqliteSchemas, fromDrizzleSqliteSnapshot, renderSqliteOp, renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaEntity } from "@maple-dev/effect-orm/schema" import { FuzzFailure, property, skip, withContext } from "./harness" // Schema plans @@ -200,6 +201,61 @@ const expectedRows = (seed: Seed, after: TablePlan): ReadonlyArray) => Promise + /** One migration, atomically: a transaction on SQLite, one batch on D1 (as Alchemy applies it). */ + readonly migrate: (statements: ReadonlyArray) => Promise + readonly all: (sql: string) => Promise>> +} + +const nodeEngine = (): Engine & { close: () => void } => { + const db = new DatabaseSync(":memory:") + const exec = async (statements: ReadonlyArray) => { + for (const sql of statements) db.exec(sql) + } + return { + exec, + migrate: (statements) => exec(["BEGIN", ...statements, "COMMIT"]), + all: async (sql) => db.prepare(sql).all().map((row) => ({ ...row })), + close: () => db.close(), + } +} + +interface D1Like { + prepare(sql: string): { all(): Promise<{ results: Array> }> } + batch(statements: ReadonlyArray): Promise +} + +/** Two D1 databases (fresh and migrated) in one Miniflare, emptied between inputs. */ +let d1: Promise<{ readonly mf: Miniflare; readonly dbs: ReadonlyArray }> | undefined +const d1Engines = async (): Promise => { + const { dbs } = await (d1 ??= (async () => { + const mf = new Miniflare({ modules: true, script: "export default { fetch() { return new Response('fuzz') } }", compatibilityDate: "2026-07-01", d1Databases: ["FRESH", "MIGRATED"] }) + return { mf, dbs: [(await mf.getD1Database("FRESH")) as unknown as D1Like, (await mf.getD1Database("MIGRATED")) as unknown as D1Like] } + })()) + const engine = (db: D1Like): Engine => ({ + exec: async (statements) => { + for (const sql of statements) await db.prepare(sql).all() + }, + migrate: async (statements) => { + if (statements.length > 0) await db.batch(statements.map((sql) => db.prepare(sql))) + }, + all: async (sql) => (await db.prepare(sql).all()).results, + }) + for (const db of dbs) { + const tables = (await db.prepare(`SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'`).all()).results + for (const { name } of tables) await db.prepare(`DROP TABLE "${name}"`).all() + } + return [engine(dbs[0]!), engine(dbs[1]!)] +} +afterAll(async () => { + if (d1 !== undefined) await (await d1).mf.dispose() +}) + // Shape interface Shape { @@ -210,89 +266,205 @@ interface Shape { * What SQLite reports, minus what legitimately depends on history: column * order (ADD COLUMN appends, documented) and auto-index names and numbering. */ -const shapeOf = (db: DatabaseSync): Shape => { - const all = (sql: string) => db.prepare(sql).all().map((row) => ({ ...row }) as Record) +const shapeOf = async (db: Engine): Promise => { const shape: Record = {} - for (const { name } of all("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")) { - const columns = all(`SELECT name, type, "notnull", dflt_value, pk FROM pragma_table_info('${name}')`).sort((a, b) => (a.name < b.name ? -1 : 1)) - const indexes = all(`SELECT name, "unique", origin, partial FROM pragma_index_list('${name}')`) - .map((index) => { - const keys = all(`SELECT seqno, name, "desc", coll FROM pragma_index_xinfo('${index.name}') WHERE "key" = 1`) - const label = String(index.name).startsWith("sqlite_autoindex_") ? `auto(${keys.map((k) => k.name).join(",")})` : index.name - return { ...index, name: label, keys } - }) - .sort((a, b) => (a.name < b.name ? -1 : 1)) - shape[name] = { columns, indexes } + for (const { name } of await db.all("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%' ORDER BY name")) { + const columns = [...(await db.all(`SELECT name, type, "notnull", dflt_value, pk FROM pragma_table_info('${name}')`))].sort((a, b) => (a.name < b.name ? -1 : 1)) + const indexes = [] + for (const index of await db.all(`SELECT name, "unique", origin, partial FROM pragma_index_list('${name}')`)) { + const keys = await db.all(`SELECT seqno, name, "desc", coll FROM pragma_index_xinfo('${index.name}') WHERE "key" = 1`) + const label = String(index.name).startsWith("sqlite_autoindex_") ? `auto(${keys.map((k) => k.name).join(",")})` : index.name + indexes.push({ ...index, name: label, keys }) + } + shape[name] = { columns, indexes: indexes.sort((a, b) => (a.name < b.name ? -1 : 1)) } } return shape } -const run = (db: DatabaseSync, statements: ReadonlyArray) => { - for (const sql of statements) db.exec(sql) +// Renames + +/** Which table (if any) the migration renames, and which of its kept columns. */ +interface RenamePlan { + readonly table: boolean + readonly column: number | null +} + +/** + * Rename one table the two schemas share, and one column both versions of it + * have, in the new schema, with the `confirm_rename` hints for them. Seeding + * and expected rows are worked out in the old names and mapped with `table` / + * `column`. + */ +const withRenames = (before: SchemaPlan, after: SchemaPlan, plan: RenamePlan | undefined) => { + const shared = after.find((t) => before.some((b) => b.name === t.name)) + const old = before.find((b) => b.name === shared?.name) + const tableTo = shared !== undefined && plan?.table === true ? `${shared.name}r` : undefined + const kept = shared === undefined || old === undefined ? [] : shared.columns.filter((c) => old.columns.some((o) => o.name === c.name)).map((c) => c.name) + const columnFrom = plan?.column != null && kept.length > 0 ? kept[plan.column % kept.length] : undefined + const table = (name: string) => (name === shared?.name && tableTo !== undefined ? tableTo : name) + const column = (tableName: string, name: string) => (tableName === shared?.name && name === columnFrom ? `${name}r` : name) + const renamed: SchemaPlan = after.map((t) => + t !== shared + ? t + : { + name: table(t.name), + columns: t.columns.map((c) => ({ ...c, name: column(t.name, c.name) })), + primaryKey: t.primaryKey.map((c) => column(t.name, c)), + indexes: t.indexes.map((index) => ({ ...index, columns: index.columns.map((c) => column(t.name, c)) })), + }, + ) + const hints = [ + ...(tableTo !== undefined ? [{ type: "confirm_rename" as const, kind: "table" as const, entity: shared!.name, to: tableTo }] : []), + ...(columnFrom !== undefined ? [{ type: "confirm_rename" as const, kind: "column" as const, entity: `${table(shared!.name)}.${columnFrom}`, to: `${columnFrom}r` }] : []), + ] + return { renamed, hints, table, column } } // The property -const migrates = async ([before, after, count, nulls]: readonly [SchemaPlan, SchemaPlan, number, ReadonlyArray]) => { +type Input = readonly [SchemaPlan, SchemaPlan, number, ReadonlyArray, RenamePlan?] + +const migrates = async ([before, after, count, nulls, renamePlan]: Input, engines: () => Promise) => { + const { renamed, hints: renameHints, table, column } = withRenames(before, after, renamePlan) const from = entitiesOf(before) - const to = entitiesOf(after) - // Every drop is confirmed: the property is about what a migration does, not the confirmation gate. - const hints = before.flatMap((table) => [ - { type: "confirm_data_loss" as const, kind: "table" as const, entity: table.name }, - ...table.columns.map((c) => ({ type: "confirm_data_loss" as const, kind: "column" as const, entity: `${table.name}.${c.name}` })), - ]) + const to = entitiesOf(renamed) + // Every drop is confirmed (under the old and the new table name): the property is about what a + // migration does, not the confirmation gate. + const hints = [ + ...renameHints, + ...before.flatMap((t) => [ + { type: "confirm_data_loss" as const, kind: "table" as const, entity: t.name }, + ...t.columns.flatMap((c) => + [...new Set([t.name, table(t.name)])].map((name) => ({ type: "confirm_data_loss" as const, kind: "column" as const, entity: `${name}.${c.name}` })), + ), + ]), + ] const diff = diffSqliteSchemas(from, to, hints) if (diff.missingHints.length > 0 || diff.unsupported.length > 0) { - throw new FuzzFailure("the diff needs more than every drop confirmed", { missingHints: diff.missingHints, unsupported: diff.unsupported }) + throw new FuzzFailure("the diff needs more than every drop and rename confirmed", { renamed, missingHints: diff.missingHints, unsupported: diff.unsupported }) } const migration = diff.ops.flatMap(renderSqliteOp) const seeds = seedRows(before, after, count, nulls) - const fresh = new DatabaseSync(":memory:") - const migrated = new DatabaseSync(":memory:") + const [fresh, migrated] = await engines() + await fresh.exec(renderSqliteSchema(to)) + await migrated.exec([...renderSqliteSchema(from), ...insertsOf(seeds)]) + const context = { before, after: renamed, renameHints, migration, seeds } try { - run(fresh, renderSqliteSchema(to)) - run(migrated, [...renderSqliteSchema(from), ...insertsOf(seeds)]) - await withContext({ before, after, migration, seeds }, () => { - try { - run(migrated, ["BEGIN", ...migration, "COMMIT"]) - } catch (error) { - throw new FuzzFailure(`the migration failed on rows that fit both schemas: ${(error as Error).message}`, {}) - } - expect(shapeOf(migrated)).toEqual(shapeOf(fresh)) - for (const seed of seeds) { - const next = after.find((t) => t.name === seed.table.name) - if (next === undefined || seed.rows.length === 0) continue - // A type change may convert a value (`'1'` to `1`); compared as text, it is the same value. - const text = (v: unknown) => (v === null || v === undefined ? null : String(v)) - const want = expectedRows(seed, next).map((row) => - Object.fromEntries(Object.entries(row).filter(([, v]) => v !== undefined).map(([k, v]) => [k, text(v)])), - ) - // Compared as a multiset: a rebuild's copy may read the old rows in index order and renumber - // their rowids, as SQLite's documented procedure (and drizzle-kit) does. - const keys = Object.keys(want[0] ?? {}) - const sorted = (rows: ReadonlyArray>) => rows.map((row) => JSON.stringify(keys.map((k) => row[k]))).sort() - const actual = migrated.prepare(`SELECT * FROM "${next.name}"`).all().map((row: Record) => Object.fromEntries(keys.map((k) => [k, text(row[k])]))) - expect(sorted(actual)).toEqual(sorted(want)) - } - }) - } finally { - fresh.close() - migrated.close() + await migrated.migrate(migration) + } catch (error) { + throw new FuzzFailure(`the migration failed on rows that fit both schemas: ${(error as Error).message}`, context) + } + const [migratedShape, freshShape] = [await shapeOf(migrated), await shapeOf(fresh)] + const tables: Array<{ name: string; want: ReadonlyArray>; rows: ReadonlyArray>; next: TablePlan }> = [] + for (const seed of seeds) { + const next = after.find((t) => t.name === seed.table.name) + if (next === undefined || seed.rows.length === 0) continue + tables.push({ name: table(next.name), want: expectedRows(seed, next), rows: await migrated.all(`SELECT * FROM "${table(next.name)}"`), next }) } + await withContext(context, () => { + expect(migratedShape).toEqual(freshShape) + for (const { name, want, rows, next } of tables) { + // A type change may convert a value (`'1'` to `1`); compared as text, it is the same value. + const text = (v: unknown) => (v === null || v === undefined ? null : String(v)) + const expected = want.map((row) => + Object.fromEntries(Object.entries(row).filter(([, v]) => v !== undefined).map(([k, v]) => [column(next.name, k), text(v)])), + ) + // Compared as a multiset: a rebuild's copy may read the old rows in index order and renumber + // their rowids, as SQLite's documented procedure (and drizzle-kit) does. + const keys = Object.keys(expected[0] ?? {}) + const sorted = (list: ReadonlyArray>) => list.map((row) => JSON.stringify(keys.map((k) => row[k]))).sort() + const actual = rows.map((row) => Object.fromEntries(keys.map((k) => [k, text(row[k])]))) + expect(sorted(actual), `rows of ${name}`).toEqual(sorted(expected)) + } + }) } +const input = fc.tuple( + schemaPlan, + schemaPlan, + fc.integer({ min: 0, max: 3 }), + fc.array(fc.boolean(), { minLength: 1, maxLength: 5 }), + fc.record({ table: fc.boolean(), column: fc.option(fc.nat(), { nil: null }) }), +) + describe("SQLite migrations", () => { - const name = "migrations reach the fresh schema and keep the rows (sqlite)" - it( - name, - () => - property( - name, - fc.tuple(schemaPlan, schemaPlan, fc.integer({ min: 0, max: 3 }), fc.array(fc.boolean(), { minLength: 1, maxLength: 5 })), - (input) => (input[0].length === 0 ? skip("no tables") : migrates(input)), - { runs: 200 }, - ), - 300_000, + const engines = { + sqlite: { runs: 200, open: async () => { + const pair = [nodeEngine(), nodeEngine()] as const + opened.push(...pair) + return pair + } }, + // D1 applies each migration as one batch, as Alchemy's D1 resource sends it. + d1: { runs: 25, open: d1Engines }, + } + const opened: Array<{ close: () => void }> = [] + afterEach(() => { + for (const db of opened.splice(0)) db.close() + }) + for (const [target, { runs, open }] of Object.entries(engines)) { + const name = `migrations reach the fresh schema and keep the rows (${target})` + it(name, () => property(name, input, (value) => (value[0].length === 0 ? skip("no tables") : migrates(value as Input, open)), { runs }), 600_000) + } +}) + +// Adopting a drizzle-kit folder + +/** + * The snapshot drizzle-kit 1.0 writes for these entities (version 7, as in + * src/kit/__fixtures__/drizzle-sqlite), including its one quirk: a primary key + * column is written `notNull: false`. + */ +const drizzleSnapshot = (entities: ReadonlyArray) => { + const keyed = new Set(entities.flatMap((e) => (e.kind === "table" ? (e.primaryKey?.columns ?? []).map((c) => `${e.name}.${c}`) : []))) + const unquote = (part: string) => /^"(.*)"$/.exec(part)?.[1]?.replaceAll('""', '"') + const ddl = entities.flatMap((e): Array> => { + switch (e.kind) { + case "table": + return [ + { name: e.name, entityType: "tables" }, + ...(e.primaryKey === null ? [] : [{ table: e.name, name: `${e.name}_pk`, columns: e.primaryKey.columns, nameExplicit: false, entityType: "pks" }]), + ] + case "column": + return [{ table: e.table, name: e.name, type: e.type, notNull: e.notNull && !keyed.has(`${e.table}.${e.name}`), autoincrement: false, default: e.default, generated: null, entityType: "columns" }] + case "index": + return [{ + table: e.table, + name: e.name, + columns: e.columns.map((part) => (unquote(part) === undefined ? { value: part, isExpression: true } : { value: unquote(part), isExpression: false })), + isUnique: e.unique, + where: null, + origin: "manual", + entityType: "indexes", + }] + case "unique": + return [{ table: e.table, name: e.name, columns: e.columns, nameExplicit: false, entityType: "uniques" }] + default: + return [] + } + }) + // drizzle-kit lists columns in table order; entities sort by key, so order them by position first. + const position = (e: Record) => (e.entityType === "columns" ? (entities.find((x) => x.kind === "column" && x.table === e.table && x.name === e.name) as { position: number }).position : 0) + return { version: "7", dialect: "sqlite", id: "fuzz", prevIds: [], ddl: [...ddl].sort((a, b) => position(a) - position(b)), renames: [] } +} + +describe("adopting a drizzle-kit SQLite folder", () => { + const name = "a drizzle-kit snapshot of any schema imports with no change to generate (sqlite)" + it(name, () => + property( + name, + schemaPlan, + (plan) => { + const entities = entitiesOf(plan) + const snapshot = drizzleSnapshot(entities) + const imported = fromDrizzleSqliteSnapshot(snapshot) + const diff = diffSqliteSchemas(imported.entities, entities) + return withContext({ plan, snapshot, imported: imported.entities, ops: diff.ops }, () => { + expect(imported.unsupported).toEqual([]) + expect(diff.ops).toEqual([]) + }) + }, + { runs: 300 }, + ), ) }) From 4cccddda7dbcccb523b9fb82c2ea71f510a313a6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:36:34 +0100 Subject: [PATCH 14/15] fix(sqlite): confirm type changes in a rebuild; rename by case without a hint From an independent review: - A rebuild that changes a kept column's type converts its values by SQLite's type affinity ('007' to 7, '1e3' to 1000) with no prompt. It now needs confirm_data_loss for that column, like a drop. - A column or table renamed by case only (Url to url) read as a drop and an add, which then collided: SQLite names ignore case. The diff now renames it, with no hint, since nothing is lost. SQLite refuses a case-only table rename in one step, so it goes through a temporary name. - The migration fuzzer renames by case without hints too, and confirms drops and type changes under old and new names. Co-Authored-By: Claude Opus 5.5 --- docs/migrations.md | 8 ++++-- src/kit/sqlite-generate.test.ts | 44 ++++++++++++++++++++++++++++-- src/schema/sqlite-diff.ts | 36 +++++++++++++++++++++++- src/schema/sqlite-ops.ts | 5 +++- tests/fuzz/migrations.fuzz.test.ts | 16 +++++++---- 5 files changed, 96 insertions(+), 13 deletions(-) diff --git a/docs/migrations.md b/docs/migrations.md index 5e8f08d..f661350 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -446,7 +446,7 @@ What each change becomes: | New table, dropped table | `CREATE TABLE`, `DROP TABLE` (confirm the data loss) | | New column | `ALTER TABLE ... ADD COLUMN`, when SQLite allows it: the column is nullable or has a literal default, and is not in the primary key or a UNIQUE constraint. Otherwise a rebuild | | Dropped column | `ALTER TABLE ... DROP COLUMN` (confirm the data loss), unless it is in the primary key or a UNIQUE constraint. Otherwise a rebuild | -| A column's type, nullability or default; the primary key; a UNIQUE column | A table rebuild | +| A column's type, nullability or default; the primary key; a UNIQUE column | A table rebuild. A new type also needs `confirm_data_loss` for the column: the copy converts values by SQLite's type affinity (`'007'` to `7`) | | New or changed index | `DROP INDEX`, `CREATE [UNIQUE] INDEX` | | Renamed table or column | `ALTER TABLE ... RENAME`, only with a `confirm_rename` hint | @@ -477,8 +477,10 @@ is the rowid and is copied. last in the database and first-to-last order differs from a fresh create. Name columns in `INSERT` and `SELECT`, as the builder does. -A rename is never guessed. Without a hint it reads as a drop plus an add, and the drop asks for -confirmation. Pass the rename instead: +A rename is never guessed, except by case: SQLite compares table and column names without case, +so `Url` to `url` (or `links` to `Links`) is the same name, renamed with no hint and no data lost. +Any other rename without a hint reads as a drop plus an add, and the drop asks for confirmation. +Pass the rename instead: ```json [ diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts index 84b6187..5806acb 100644 --- a/src/kit/sqlite-generate.test.ts +++ b/src/kit/sqlite-generate.test.ts @@ -119,7 +119,9 @@ it.each(rebuilds)("rebuilds the table for %s, keeps its rows and recreates its i const index = `indexes: [SQLite.index("links_url", ["url"])],` await generateFrom(links(before, index)) const v2 = links(after, index) - const second = await generateFrom(v2) + // A new column type rewrites values by SQLite's affinity rules, so it is confirmed like a drop. + const hints = before.includes("hits") ? [{ type: "confirm_data_loss" as const, kind: "column" as const, entity: "links.hits" }] : [] + const second = await generateFrom(v2, { hints }) expect(second.plan.map((p) => p.label)).toEqual(["rebuild", "metadata"]) expect(second.plan[0]!.sql).toEqual([ "PRAGMA foreign_keys=OFF", @@ -175,13 +177,49 @@ it("creates and drops indexes without touching the table", async () => { it("asks to confirm a column a rebuild leaves behind", async () => { await generateFrom(links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text)`)) const v2 = links(`id: SQLite.text, url: SQLite.integer`) - await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints", details: [expect.stringContaining("links.note")] }) - const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.note" }] }) + // `note` is left behind, and `url` changes type (text to integer), which rewrites its values. + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints", details: [expect.stringMatching(/links\.note.*links\.url/)] }) + const second = await generateFrom(v2, { + hints: [ + { type: "confirm_data_loss", kind: "column", entity: "links.note" }, + { type: "confirm_data_loss", kind: "column", entity: "links.url" }, + ], + }) expect(second.plan.map((p) => p.label)).toEqual(["rebuild"]) expect(second.plan[0]!.sql[2]).toBe('INSERT INTO "__new_links"("id", "url") SELECT "id", "url" FROM "links"') await expectMigratedToEqualFresh(v2) }) +it("asks before a type change rewrites a column's values", async () => { + await generateFrom(links(`id: SQLite.text, code: SQLite.text`)) + const v2 = links(`id: SQLite.text, code: SQLite.integer`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints", details: [expect.stringContaining("links.code")] }) + await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.code" }] }) + // What the hint confirms: SQLite's integer affinity reads '007' as 7. + const seed = [`INSERT INTO "links" ("id", "code") VALUES ('a', '007')`] + expect(await rowsAfter(allStatements(seed), 'SELECT "code" FROM "links"')).toEqual([{ code: 7 }]) +}) + +it("renames a table or column whose name changes only in case, with no hint and no data lost", async () => { + await generateFrom(links(`id: SQLite.text, Url: SQLite.text`)) + const v2 = ` +export const Links = SQLite.table("Links", { + columns: { id: SQLite.text, url: SQLite.text }, + primaryKey: ["id"], +})` + const second = await generateFrom(v2) + // SQLite refuses `links` to `Links` in one step, so it goes through a temporary name. + expect(second.plan.flatMap((p) => p.sql)).toEqual([ + 'ALTER TABLE "links" RENAME TO "__rename_Links"', + 'ALTER TABLE "__rename_Links" RENAME TO "Links"', + 'ALTER TABLE "Links" RENAME COLUMN "Url" TO "url"', + ]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements([`INSERT INTO "links" ("id", "Url") VALUES ('a', 'https://a')`]), 'SELECT "id", "url" FROM "Links"')).toEqual([ + { id: "a", url: "https://a" }, + ]) +}) + it("drops a table only once the data loss is confirmed", async () => { await generateFrom(`${links(`id: SQLite.text, url: SQLite.text`)}\nexport const Old = SQLite.table("old", { columns: { id: SQLite.text }, indexes: [SQLite.index("old_id", ["id"])] })`) const v2 = links(`id: SQLite.text, url: SQLite.text`) diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts index aa82aaf..c6d89ed 100644 --- a/src/schema/sqlite-diff.ts +++ b/src/schema/sqlite-diff.ts @@ -40,6 +40,9 @@ export const diffSqliteSchemas = ( next: ReadonlyArray, hints: ReadonlyArray = [], ): DiffResult => { + // SQLite compares table and column names without case: `Name` to `name` is the same column, + // renamed, never a drop and an add (which would collide). No hint needed: nothing is lost. + hints = [...hints, ...caseOnlyRenames(original, next, hints)] const given = new Set(hints.map(hintKey)) const ops: Array = [] const missingHints: Array = [] @@ -144,6 +147,8 @@ const diffTable = ( const added = after.columns.filter((c) => !old.has(c.name)) const dropped = before.columns.filter((c) => !kept.has(c.name)) const changed = after.columns.filter((c) => old.has(c.name) && !same(definition(old.get(c.name)!), definition(c))) + // A new type rewrites the column's values by SQLite's affinity rules ('007' becomes 7): confirm it like a drop. + const retyped = changed.filter((c) => old.get(c.name)!.type !== c.type) const needsRebuild = changed.length > 0 || !same(before.table.primaryKey, after.table.primaryKey) || @@ -159,13 +164,42 @@ const diffTable = ( uniques: after.uniques, copy: after.columns.filter((c) => old.has(c.name)).map((c) => c.name), } - confirm(dropped.map((c) => dropHint(c.name)), rebuild) + confirm([...dropped, ...retyped].map((c) => dropHint(c.name)), rebuild) return } for (const column of added) ops.push({ op: "add_column", column }) for (const column of dropped) confirm(dropHint(column.name), { op: "drop_column", table: name, name: column.name }) } +/** + * `confirm_rename` hints for names that differ only in case, tables first: a + * next table or column with no exact match whose name, ignoring case, is an + * old one's that is gone. Columns are matched in the table's new name. + */ +const caseOnlyRenames = (prev: ReadonlyArray, next: ReadonlyArray, hints: ReadonlyArray): Array => { + const renames: Array = [] + const tables = (entities: ReadonlyArray) => pick(entities, "table").map((t) => t.name) + const match = (before: ReadonlyArray, after: ReadonlyArray) => + after.flatMap((name) => { + if (before.includes(name)) return [] + const old = before.find((b) => b.toLowerCase() === name.toLowerCase() && !after.includes(b)) + return old === undefined ? [] : [[old, name] as const] + }) + const hinted = new Map(hints.flatMap((h) => (h.type === "confirm_rename" && h.kind === "table" ? [[h.to, h.entity] as const] : []))) + const tableRenames = match(tables(prev).filter((t) => ![...hinted.values()].includes(t)), tables(next).filter((t) => !hinted.has(t))) + for (const [from, to] of tableRenames) renames.push({ type: "confirm_rename", kind: "table", entity: from, to }) + const source = new Map([...hinted, ...tableRenames.map(([from, to]) => [to, from] as const)]) + const columnsOf = (entities: ReadonlyArray, table: string) => pick(entities, "column").filter((c) => c.table === table).map((c) => c.name) + const hintedColumns = new Set(hints.flatMap((h) => (h.type === "confirm_rename" && h.kind === "column" ? [h.entity] : []))) + for (const table of tables(next)) { + const before = columnsOf(prev, source.get(table) ?? table) + for (const [from, to] of match(before, columnsOf(next, table))) { + if (!hintedColumns.has(`${table}.${from}`)) renames.push({ type: "confirm_rename", kind: "column", entity: `${table}.${from}`, to }) + } + } + return renames +} + const byName = (uniques: ReadonlyArray) => [...uniques].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) diff --git a/src/schema/sqlite-ops.ts b/src/schema/sqlite-ops.ts index 32edcd6..7a396e3 100644 --- a/src/schema/sqlite-ops.ts +++ b/src/schema/sqlite-ops.ts @@ -75,7 +75,10 @@ export const renderSqliteOp = (op: SqliteMigrationOp): ReadonlyArray => case "drop_table": return [`DROP TABLE IF EXISTS ${sqliteIdent(op.name)}`] case "rename_table": - return [`ALTER TABLE ${sqliteIdent(op.from)} RENAME TO ${sqliteIdent(op.to)}`] + // SQLite refuses a rename to the same name in another case ("already another table"): go through a temporary one. + return op.from.toLowerCase() === op.to.toLowerCase() + ? [`ALTER TABLE ${sqliteIdent(op.from)} RENAME TO ${sqliteIdent(`__rename_${op.to}`)}`, `ALTER TABLE ${sqliteIdent(`__rename_${op.to}`)} RENAME TO ${sqliteIdent(op.to)}`] + : [`ALTER TABLE ${sqliteIdent(op.from)} RENAME TO ${sqliteIdent(op.to)}`] case "add_column": return [`ALTER TABLE ${sqliteIdent(op.column.table)} ADD COLUMN ${renderSqliteColumnDefinition(op.column)}`] case "drop_column": diff --git a/tests/fuzz/migrations.fuzz.test.ts b/tests/fuzz/migrations.fuzz.test.ts index b3accd9..ebbc3f0 100644 --- a/tests/fuzz/migrations.fuzz.test.ts +++ b/tests/fuzz/migrations.fuzz.test.ts @@ -287,6 +287,8 @@ const shapeOf = async (db: Engine): Promise => { interface RenamePlan { readonly table: boolean readonly column: number | null + /** Rename by case only (`t0` to `T0`), with no hint: SQLite names ignore case, so the diff must see a rename. */ + readonly caseOnly?: boolean } /** @@ -298,11 +300,12 @@ interface RenamePlan { const withRenames = (before: SchemaPlan, after: SchemaPlan, plan: RenamePlan | undefined) => { const shared = after.find((t) => before.some((b) => b.name === t.name)) const old = before.find((b) => b.name === shared?.name) - const tableTo = shared !== undefined && plan?.table === true ? `${shared.name}r` : undefined + const caseOnly = plan?.caseOnly === true + const tableTo = shared !== undefined && plan?.table === true ? (caseOnly ? shared.name.toUpperCase() : `${shared.name}r`) : undefined const kept = shared === undefined || old === undefined ? [] : shared.columns.filter((c) => old.columns.some((o) => o.name === c.name)).map((c) => c.name) const columnFrom = plan?.column != null && kept.length > 0 ? kept[plan.column % kept.length] : undefined const table = (name: string) => (name === shared?.name && tableTo !== undefined ? tableTo : name) - const column = (tableName: string, name: string) => (tableName === shared?.name && name === columnFrom ? `${name}r` : name) + const column = (tableName: string, name: string) => (tableName === shared?.name && name === columnFrom ? (caseOnly ? name.toUpperCase() : `${name}r`) : name) const renamed: SchemaPlan = after.map((t) => t !== shared ? t @@ -313,7 +316,7 @@ const withRenames = (before: SchemaPlan, after: SchemaPlan, plan: RenamePlan | u indexes: t.indexes.map((index) => ({ ...index, columns: index.columns.map((c) => column(t.name, c)) })), }, ) - const hints = [ + const hints = caseOnly ? [] : [ ...(tableTo !== undefined ? [{ type: "confirm_rename" as const, kind: "table" as const, entity: shared!.name, to: tableTo }] : []), ...(columnFrom !== undefined ? [{ type: "confirm_rename" as const, kind: "column" as const, entity: `${table(shared!.name)}.${columnFrom}`, to: `${columnFrom}r` }] : []), ] @@ -334,8 +337,11 @@ const migrates = async ([before, after, count, nulls, renamePlan]: Input, engine ...renameHints, ...before.flatMap((t) => [ { type: "confirm_data_loss" as const, kind: "table" as const, entity: t.name }, + // Old and new names: a drop is named in the table's new name, a type change also in the column's. ...t.columns.flatMap((c) => - [...new Set([t.name, table(t.name)])].map((name) => ({ type: "confirm_data_loss" as const, kind: "column" as const, entity: `${name}.${c.name}` })), + [...new Set([t.name, table(t.name)])].flatMap((name) => + [...new Set([c.name, column(t.name, c.name)])].map((col) => ({ type: "confirm_data_loss" as const, kind: "column" as const, entity: `${name}.${col}` })), + ), ), ]), ] @@ -385,7 +391,7 @@ const input = fc.tuple( schemaPlan, fc.integer({ min: 0, max: 3 }), fc.array(fc.boolean(), { minLength: 1, maxLength: 5 }), - fc.record({ table: fc.boolean(), column: fc.option(fc.nat(), { nil: null }) }), + fc.record({ table: fc.boolean(), column: fc.option(fc.nat(), { nil: null }), caseOnly: fc.boolean() }), ) describe("SQLite migrations", () => { From 7c8e7e151bf9e6931699141b040e1547bad3fc84 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 20:03:13 +0100 Subject: [PATCH 15/15] test(fuzz): the D1 migration engine retries transport errors only Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/migrations.fuzz.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/fuzz/migrations.fuzz.test.ts b/tests/fuzz/migrations.fuzz.test.ts index ebbc3f0..882c80d 100644 --- a/tests/fuzz/migrations.fuzz.test.ts +++ b/tests/fuzz/migrations.fuzz.test.ts @@ -19,6 +19,7 @@ import { Miniflare } from "miniflare" import { afterAll, afterEach, describe, expect, it } from "vitest" import * as SQLite from "@maple-dev/effect-orm/sqlite" import { diffSqliteSchemas, fromDrizzleSqliteSnapshot, renderSqliteOp, renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaEntity } from "@maple-dev/effect-orm/schema" +import { retryTransport } from "../sqlite-support" import { FuzzFailure, property, skip, withContext } from "./harness" // Schema plans @@ -239,12 +240,12 @@ const d1Engines = async (): Promise => { })()) const engine = (db: D1Like): Engine => ({ exec: async (statements) => { - for (const sql of statements) await db.prepare(sql).all() + for (const sql of statements) await retryTransport(() => db.prepare(sql).all()) }, migrate: async (statements) => { - if (statements.length > 0) await db.batch(statements.map((sql) => db.prepare(sql))) + if (statements.length > 0) await retryTransport(() => db.batch(statements.map((sql) => db.prepare(sql)))) }, - all: async (sql) => (await db.prepare(sql).all()).results, + all: async (sql) => retryTransport(async () => (await db.prepare(sql).all()).results), }) for (const db of dbs) { const tables = (await db.prepare(`SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'`).all()).results