diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md new file mode 100644 index 0000000..628a175 --- /dev/null +++ b/.changeset/quiet-sqlite-dialects.md @@ -0,0 +1,8 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Add the `/sqlite` query entry with SQLite codecs, functions and positional bindings. Use +`sqliteDialect` with an Effect SQLite SqlClient, or `d1Dialect` for Cloudflare D1's binding limit +and explicit refusal of interactive transactions. `table` is a query schema only: SQLite DDL and +migration generation are not implemented yet. diff --git a/.changeset/sqlite-generate.md b/.changeset/sqlite-generate.md new file mode 100644 index 0000000..0630148 --- /dev/null +++ b/.changeset/sqlite-generate.md @@ -0,0 +1,10 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`effect-orm generate` supports `dialect: "sqlite"`. It diffs `SQLite.table` definitions against +the migrations folder and writes `_/migration.sql` plus `snapshot.json`, the +drizzle-kit 1.0 layout that Alchemy's D1 resource applies. `ALTER TABLE` adds, drops and renames +where SQLite allows it; other column, primary key and UNIQUE changes rebuild the table. Renames +need a `confirm_rename` hint. `generate --baseline --from-drizzle` adopts a drizzle-kit 1.0 +SQLite folder. Snapshots gain a `sqlite` dialect. `/migrate` refuses SQLite folders for now. diff --git a/.changeset/sqlite-table-ddl.md b/.changeset/sqlite-table-ddl.md new file mode 100644 index 0000000..e38de76 --- /dev/null +++ b/.changeset/sqlite-table-ddl.md @@ -0,0 +1,10 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`SQLite.table(name, { columns, primaryKey, indexes })` declares a SQLite table with its DDL: +`NOT NULL` from the column type, single and composite primary keys, literal and expression +defaults, unique columns, and named indexes and unique indexes. `sqliteEntitiesOf` and +`renderSqliteSchema` from `/schema` check a schema and write its `CREATE TABLE` and +`CREATE INDEX` statements. `table(name, columns, options?)` and +`table(name, { external: true, columns })` stay query-only. SQLite migrations are not generated yet. diff --git a/README.md b/README.md index 4863efe..0e9299e 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ > **Alpha.** The API is still changing and minor releases may break it. Pin an exact version, > read the [changelog](./CHANGELOG.md) before upgrading, and expect rough edges. -Type-safe ClickHouse and Postgres queries, result decoding, and reproducible benchmarks for Effect and TypeScript. +Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproducible benchmarks for Effect and TypeScript. Formerly `@maple-dev/effect-clickhouse`. +SQLite and Cloudflare D1 use the same builder and row decoder through the new +[`/sqlite` entry](./docs/sqlite.md), which also writes SQLite table DDL. `effect-orm generate` writes SQLite migrations; effect-orm does not apply them yet. + [Read the documentation](https://effect-clickhouse.maple.dev) · [Getting started](./docs/getting-started.md) · [Recipes](./docs/recipes.md) diff --git a/bun.lock b/bun.lock index db92323..a8a9d5a 100644 --- a/bun.lock +++ b/bun.lock @@ -10,13 +10,16 @@ "@effect/language-service": "^0.87.3", "@effect/platform-bun": "4.0.0", "@effect/sql-clickhouse": "4.0.0", + "@effect/sql-d1": "4.0.0", "@effect/sql-pglite": "4.0.0", + "@effect/sql-sqlite-node": "4.0.0", "@effect/vitest": "4.0.0", "@electric-sql/pglite": "0.5.8", "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", "fast-check": "^4.9.0", + "miniflare": "4.20260730.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9", @@ -68,6 +71,20 @@ "@clickhouse/client-common": ["@clickhouse/client-common@1.21.0", "", {}, "sha512-UjvA32aaLKbQ5U735qAGdzohrOrfU8zFoOWIOWorWQk+7UzHZxq6pf4Uh/wQ08zAMb9cmkd9h9N1FV8zP8UCOg=="], + "@cloudflare/workerd-darwin-64": ["@cloudflare/workerd-darwin-64@1.20260730.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA=="], + + "@cloudflare/workerd-darwin-arm64": ["@cloudflare/workerd-darwin-arm64@1.20260730.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ=="], + + "@cloudflare/workerd-linux-64": ["@cloudflare/workerd-linux-64@1.20260730.1", "", { "os": "linux", "cpu": "x64" }, "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA=="], + + "@cloudflare/workerd-linux-arm64": ["@cloudflare/workerd-linux-arm64@1.20260730.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ=="], + + "@cloudflare/workerd-windows-64": ["@cloudflare/workerd-windows-64@1.20260730.1", "", { "os": "win32", "cpu": "x64" }, "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ=="], + + "@cloudflare/workers-types": ["@cloudflare/workers-types@5.20261024.1", "", {}, "sha512-sNhbgBsuVvmumSurgCcIrXShqACPfRt/XlYFstBeANu5BHEJhXW+e7PmHxDNYRXxKYZV57toUfkLwFp8/0SZKA=="], + + "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@effect/language-service": ["@effect/language-service@0.87.3", "", { "bin": { "effect-language-service": "cli.js" } }, "sha512-g1OmRBZF5chbW9QMNoirPo5Bb4zzU2HfP88tR7/VSZEUxHJ1/qVb8AQ0kUh2ggctLH1I715yCwP4gkw56PTjpQ=="], "@effect/platform-bun": ["@effect/platform-bun@4.0.0", "", { "dependencies": { "@effect/platform-node-shared": "^4.0.0" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-g9O41Sub/+JKfLjvBNEoqR1Pg6wtc2bbYzv7Kw8YsMihbt/ypCn7ZoydEXaHvTTUvbVrXOQbVFlfplyo84Clug=="], @@ -78,14 +95,78 @@ "@effect/sql-clickhouse": ["@effect/sql-clickhouse@4.0.0", "", { "dependencies": { "@clickhouse/client": "^1.23.1" }, "peerDependencies": { "@effect/platform-node": "^4.0.0", "effect": "^4.0.0" } }, "sha512-KthAxSJAW4VR9YyFsgOwwWX2ym6aYMFEJkzyxJExzcGFiUbFMK7hsbwzagiBKJAr/N1gGWQYiZP2fZgkgruiuA=="], + "@effect/sql-d1": ["@effect/sql-d1@4.0.0", "", { "dependencies": { "@cloudflare/workers-types": "^5.20260926.1" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-pmkzncZCGD6CLyvc89keEBU85h+gHVCu/LKQnThd+ruYSUbG0R9jYA+lnk3R8Djp0wQW6SbsbmHQuhopIYyo3Q=="], + "@effect/sql-pglite": ["@effect/sql-pglite@4.0.0", "", { "dependencies": { "@electric-sql/pglite": "^0.5.8" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-BTlCPSipKMHZJEoJ94rR4f8Gx8cv8tLpZeHc3sajg2ko/2GY8s6CyycSra3Z6GLFUhcivRTYQ9Zc5LThz64qfQ=="], + "@effect/sql-sqlite-node": ["@effect/sql-sqlite-node@4.0.0", "", { "peerDependencies": { "effect": "^4.0.0" } }, "sha512-xyUSLY7v55iMgGxrjFrTo5ngixgQ3jLmOQ+AX8CCazO/caxuMM0oq8zb974FAcvQHRrpMEWU7Vnoo+rxct7PAQ=="], + "@effect/vitest": ["@effect/vitest@4.0.0", "", { "peerDependencies": { "effect": "^4.0.0", "vitest": ">=5.0.0 <6.0.0" } }, "sha512-kGElCPFGtP+CgMCwV/HjJWqtE3zYo6I1+8MSA54l+EI24lPU9p2Efa+TBp1jfkg9j711KKzEqgSAy3VzTsb76Q=="], "@electric-sql/pglite": ["@electric-sql/pglite@0.5.8", "", {}, "sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA=="], + + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.1" }, "os": "darwin", "cpu": "arm64" }, "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.1" }, "os": "darwin", "cpu": "x64" }, "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw=="], + + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "os": "freebsd" }, "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.1", "", { "os": "linux", "cpu": "none" }, "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.1" }, "os": "linux", "cpu": "arm" }, "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.1" }, "os": "linux", "cpu": "ppc64" }, "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.1" }, "os": "linux", "cpu": "none" }, "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.1" }, "os": "linux", "cpu": "s390x" }, "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.2", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw=="], + + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "cpu": "none" }, "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.2", "", { "os": "win32", "cpu": "x64" }, "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="], + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], + "@manypkg/find-root": ["@manypkg/find-root@3.1.0", "", { "dependencies": { "@manypkg/tools": "^2.1.0" } }, "sha512-BcSqCyKhBVZ5YkSzOiheMCV41kqAFptW6xGqYSTjkVTl9XQpr+pqHhwgGCOHQtjDCv7Is6EFyA14Sm5GVbVABA=="], "@manypkg/get-packages": ["@manypkg/get-packages@3.1.0", "", { "dependencies": { "@manypkg/find-root": "^3.1.0", "@manypkg/tools": "^2.1.0" } }, "sha512-0TbBVyvPrP7xGYBI/cP8UP+yl/z+HtbTttAD7FMAJgn/kXOTwh5/60TsqP9ZYY710forNfyV0N8P/IE/ujGZJg=="], @@ -96,6 +177,12 @@ "@pnpm/deps.graph-sequencer": ["@pnpm/deps.graph-sequencer@1100.0.1", "", {}, "sha512-pOr5+q1fLYKwFN3LAJuGZEnfXDcQ73zqgDHMtGy+K+uIoUqyY+6MeDCWFwfu+4EFuq76I5EPFofoNAI+Bmmq4A=="], + "@poppinss/colors": ["@poppinss/colors@4.1.6", "", { "dependencies": { "kleur": "^4.1.5" } }, "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg=="], + + "@poppinss/dumper": ["@poppinss/dumper@0.6.5", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@sindresorhus/is": "^7.0.2", "supports-color": "^10.0.0" } }, "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw=="], + + "@poppinss/exception": ["@poppinss/exception@1.2.3", "", {}, "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw=="], + "@quansync/fs": ["@quansync/fs@1.1.0", "", { "dependencies": { "quansync": "^1.0.0" } }, "sha512-qAPG/t3HqML1TlN7sY/pTbEjzFVAKsMjNNMGheyDosro+kT4iw2KCUoHcVdmliwWjorm4elZbgNQyU2eD97eDg=="], "@redis/bloom": ["@redis/bloom@6.3.0", "", { "peerDependencies": { "@redis/client": "^6.3.0" } }, "sha512-NQ5poYpZr0jv6zazN1cy6JY5kIipKFZ3R9GTL4kOUTX3a9YAfY29U8bsE4lWhVQEM+G4V0BXVC57mubIdDijeA=="], @@ -140,6 +227,10 @@ "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "@sindresorhus/is": ["@sindresorhus/is@7.2.0", "", {}, "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw=="], + + "@speed-highlight/core": ["@speed-highlight/core@1.2.24", "", {}, "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], @@ -266,6 +357,8 @@ "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], + "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], + "defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -276,6 +369,8 @@ "empathic": ["empathic@2.1.0", "", {}, "sha512-AnfC1ATldl49/cvZdLPDjBfrRNwbDO05aibiOtzQu3qtlbJtomNLhF30HEtn/7iBz50dlMECqATo3fG0LrdEgw=="], + "error-stack-parser-es": ["error-stack-parser-es@1.0.5", "", {}, "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], @@ -308,6 +403,8 @@ "jsonc-parser": ["jsonc-parser@3.3.1", "", {}, "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ=="], + "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], + "launch-editor": ["launch-editor@2.14.2", "", { "dependencies": { "picocolors": "^1.1.1", "shell-quote": "^1.10.0" } }, "sha512-NHJEb30MGBCkmZXLZwz25t37QR4aY03ZzPYUJXOJaOa92uWwExI9SBQUuIznBYc1xkD7cJNGX7oc3Nl3sq2kQQ=="], "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], @@ -336,6 +433,8 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "miniflare": ["miniflare@4.20260730.0", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", "undici": "7.28.0", "workerd": "1.20260730.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-1Z9SB9r/o//80UA02Re3QhtcecSHAyAjf5EcKBfQVlQrCg7Miy79hl2PvtkwFLIaJ5rcrOPdDcRr577okwZPsg=="], + "nanoid": ["nanoid@3.3.19", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug=="], "obug": ["obug@2.2.1", "", {}, "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q=="], @@ -364,6 +463,8 @@ "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "sharp": ["sharp@0.35.2", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.4" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.2", "@img/sharp-darwin-x64": "0.35.2", "@img/sharp-freebsd-wasm32": "0.35.2", "@img/sharp-libvips-darwin-arm64": "1.3.1", "@img/sharp-libvips-darwin-x64": "1.3.1", "@img/sharp-libvips-linux-arm": "1.3.1", "@img/sharp-libvips-linux-arm64": "1.3.1", "@img/sharp-libvips-linux-ppc64": "1.3.1", "@img/sharp-libvips-linux-riscv64": "1.3.1", "@img/sharp-libvips-linux-s390x": "1.3.1", "@img/sharp-libvips-linux-x64": "1.3.1", "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", "@img/sharp-libvips-linuxmusl-x64": "1.3.1", "@img/sharp-linux-arm": "0.35.2", "@img/sharp-linux-arm64": "0.35.2", "@img/sharp-linux-ppc64": "0.35.2", "@img/sharp-linux-riscv64": "0.35.2", "@img/sharp-linux-s390x": "0.35.2", "@img/sharp-linux-x64": "0.35.2", "@img/sharp-linuxmusl-arm64": "0.35.2", "@img/sharp-linuxmusl-x64": "0.35.2", "@img/sharp-webcontainers-wasm32": "0.35.2", "@img/sharp-win32-arm64": "0.35.2", "@img/sharp-win32-ia32": "0.35.2", "@img/sharp-win32-x64": "0.35.2" } }, "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w=="], + "shell-quote": ["shell-quote@1.12.0", "", {}, "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q=="], "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], @@ -376,6 +477,8 @@ "std-env": ["std-env@4.3.0", "", {}, "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag=="], + "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], "tinyexec": ["tinyexec@1.3.1", "", {}, "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA=="], @@ -388,11 +491,13 @@ "tsdown": ["tsdown@0.22.14", "", { "dependencies": { "ansis": "^4.3.1", "cac": "^7.0.0", "defu": "^6.1.7", "empathic": "^2.0.1", "hookable": "^6.1.1", "import-without-cache": "^0.4.0", "obug": "^2.1.4", "picomatch": "^4.0.5", "rolldown": "~1.2.0", "rolldown-plugin-dts": "^0.27.13", "tinyexec": "^1.2.4", "tinyglobby": "^0.2.17", "tree-kill": "^1.2.2", "unconfig-core": "^7.5.0", "verkit": "^0.3.0" }, "peerDependencies": { "@arethetypeswrong/core": "^0.18.1", "@tsdown/css": "0.22.14", "@tsdown/exe": "0.22.14", "@vitejs/devtools": "*", "publint": "^0.3.8", "tsx": "*", "typescript": "^5.0.0 || ^6.0.0 || ^7.0.0", "unplugin-unused": "^0.5.0", "unrun": "*" }, "optionalPeers": ["@arethetypeswrong/core", "@tsdown/css", "@tsdown/exe", "@vitejs/devtools", "publint", "tsx", "typescript", "unplugin-unused", "unrun"], "bin": { "tsdown": "./dist/run.mjs" } }, "sha512-ule7Y+fsAN2iZbLDoo7C4KYljFJNJJ+fLshyn+9gozeTspVersWHxwdGB+Dm2hzA38s6muFnUTl0jK3vJm9ifQ=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "unconfig-core": ["unconfig-core@7.5.0", "", { "dependencies": { "@quansync/fs": "^1.0.0", "quansync": "^1.0.0" } }, "sha512-Su3FauozOGP44ZmKdHy2oE6LPjk51M/TRRjHv2HNCWiDvfvCoxC2lno6jevMA91MYAdCdwP05QnWdWpSbncX/w=="], - "undici": ["undici@8.11.2", "", {}, "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ=="], + "undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -404,16 +509,26 @@ "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - "ws": ["ws@8.22.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg=="], + "workerd": ["workerd@1.20260730.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260730.1", "@cloudflare/workerd-darwin-arm64": "1.20260730.1", "@cloudflare/workerd-linux-64": "1.20260730.1", "@cloudflare/workerd-linux-arm64": "1.20260730.1", "@cloudflare/workerd-windows-64": "1.20260730.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA=="], + + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], + "youch": ["youch@4.1.0-beta.10", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@poppinss/dumper": "^0.6.4", "@speed-highlight/core": "^1.2.7", "cookie": "^1.0.2", "youch-core": "^0.3.3" } }, "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ=="], + + "youch-core": ["youch-core@0.3.3", "", { "dependencies": { "@poppinss/exception": "^1.2.2", "error-stack-parser-es": "^1.0.5" } }, "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA=="], + "yuku-ast": ["yuku-ast@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7" } }, "sha512-h6+4bDfyootiMB9vckk5uKo5r5j0GHrkr17FQTDNfEsFT3DWlN9uu1HJwQwc64pgmLCI945fWM3lbTIqxjT3GQ=="], "yuku-codegen": ["yuku-codegen@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7" }, "optionalDependencies": { "@yuku-codegen/binding-android-arm64": "0.8.7", "@yuku-codegen/binding-darwin-arm64": "0.8.7", "@yuku-codegen/binding-darwin-x64": "0.8.7", "@yuku-codegen/binding-freebsd-x64": "0.8.7", "@yuku-codegen/binding-linux-arm-gnu": "0.8.7", "@yuku-codegen/binding-linux-arm-musl": "0.8.7", "@yuku-codegen/binding-linux-arm64-gnu": "0.8.7", "@yuku-codegen/binding-linux-arm64-musl": "0.8.7", "@yuku-codegen/binding-linux-x64-gnu": "0.8.7", "@yuku-codegen/binding-linux-x64-musl": "0.8.7", "@yuku-codegen/binding-win32-arm64": "0.8.7", "@yuku-codegen/binding-win32-x64": "0.8.7" } }, "sha512-adwDZSh8oVDzhE6Du9PwVWxcOxeV0e2EVhUuMKWfhSY4wkrDq9eqixlxFF3l/XGUV1E7UFzhpz9393MUumkyNw=="], "yuku-parser": ["yuku-parser@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7", "yuku-ast": "^0.8.7" }, "optionalDependencies": { "@yuku-parser/binding-android-arm64": "0.8.7", "@yuku-parser/binding-darwin-arm64": "0.8.7", "@yuku-parser/binding-darwin-x64": "0.8.7", "@yuku-parser/binding-freebsd-x64": "0.8.7", "@yuku-parser/binding-linux-arm-gnu": "0.8.7", "@yuku-parser/binding-linux-arm-musl": "0.8.7", "@yuku-parser/binding-linux-arm64-gnu": "0.8.7", "@yuku-parser/binding-linux-arm64-musl": "0.8.7", "@yuku-parser/binding-linux-x64-gnu": "0.8.7", "@yuku-parser/binding-linux-x64-musl": "0.8.7", "@yuku-parser/binding-win32-arm64": "0.8.7", "@yuku-parser/binding-win32-x64": "0.8.7" } }, "sha512-vRD9nwt4L3aYpxNqeSC4WqLv58xrXef0Ong1Mc45CTXTIpvLafx7JO05sczmQZwdLEZvywrLOGdNC5+Rp5N1BQ=="], + "@effect/platform-node/undici": ["undici@8.11.2", "", {}, "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ=="], + + "@effect/platform-node-shared/ws": ["ws@8.22.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg=="], + "@effect/sql-clickhouse/@clickhouse/client": ["@clickhouse/client@1.23.1", "", {}, "sha512-vs3/Zc1dHvT171btW5nMoPsPCJ6QVJ5pp7obxzO5sjqwFx/jjz9wwCAqcFOdc2DhprugDBaVn+4dVY8hG3A9nw=="], } } diff --git a/docs/README.md b/docs/README.md index 7a61d11..000b160 100644 --- a/docs/README.md +++ b/docs/README.md @@ -99,3 +99,7 @@ The optional `@maple-dev/effect-orm/benchmark` entry point and bundled `ch-bench` CLI measure real queries, compare fixed workloads, and save evidence. See [Benchmarking](./benchmarking.md) and the [agent playbook](./benchmark-agent.md). The SQL builders remain driver-free. + +## SQLite and D1 + +[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, table DDL, Effect SQL drivers, tenant checks and transaction limits. `effect-orm generate` writes SQLite migrations; effect-orm does not apply them yet. diff --git a/docs/migrations.md b/docs/migrations.md index 67f9e26..694230e 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,6 +1,6 @@ # Schema and migrations -Every table is declared with its DDL: `table` from `/clickhouse` or `/postgres` carries the +Every table is declared with its DDL: `table` from `/clickhouse`, `/postgres` or `/sqlite` carries the engine, keys, indexes, and defaults beside the columns the query builder reads. You can ignore that and manage the database elsewhere, or let three opt-in entry points turn the definitions into migrations: @@ -17,6 +17,7 @@ check and folder loading are shared; table definitions, the diff, the DDL, the r drift detection are per database, because the two differ where it matters: ClickHouse DDL is not transactional and cannot change most things in place, and Postgres DDL is and can. The sections below describe ClickHouse first; [Postgres](#postgres) covers what differs. +[SQLite](#sqlite) has `generate` only: another tool applies its migrations for now. ## Defining tables @@ -420,3 +421,74 @@ drizzle-kit recorded, so every place they disagree (a constraint name, a default op to accept or fix. `verify` against the baseline also finds objects the database has and drizzle-kit's snapshot does not, such as a table a hand-written migration created and nothing dropped. + +## SQLite + +Set `dialect: "sqlite"` and export `SQLite.table` values with their DDL from the schema module: + +```ts title="sqlite-kit-config.ts" +import { defineConfig } from "@maple-dev/effect-orm/kit" + +export default defineConfig({ dialect: "sqlite", schema: "./src/db/schema.ts", out: "./migrations" }) +``` + +`effect-orm generate` writes `_/migration.sql` and `snapshot.json`, the +drizzle-kit 1.0 layout. Statements are split by `--> statement-breakpoint`, each under a comment +naming its label. A SQLite migration is always SQL; there is no `migration.json` for it. Tools +that apply drizzle-kit folders apply it as it is, for example Alchemy's +`Cloudflare.D1.Database({ migrations: "./migrations" })`, which runs each folder's statements as one +D1 batch. `snapshot.json` is for `generate` and `check`; those tools ignore it. + +What each change becomes: + +| Change | SQL | +| --- | --- | +| New table, dropped table | `CREATE TABLE`, `DROP TABLE` (confirm the data loss) | +| New column | `ALTER TABLE ... ADD COLUMN`, when SQLite allows it: the column is nullable or has a literal default, and is not in the primary key or a UNIQUE constraint. Otherwise a rebuild | +| Dropped column | `ALTER TABLE ... DROP COLUMN` (confirm the data loss), unless it is in the primary key or a UNIQUE constraint. Otherwise a rebuild | +| A column's type, nullability or default; the primary key; a UNIQUE column | A table rebuild | +| New or changed index | `DROP INDEX`, `CREATE [UNIQUE] INDEX` | +| Renamed table or column | `ALTER TABLE ... RENAME`, only with a `confirm_rename` hint | + +A rebuild follows [SQLite's procedure](https://www.sqlite.org/lang_altertable.html#otheralter) +the way drizzle-kit writes it: `PRAGMA foreign_keys=OFF`, `CREATE TABLE "__new_"` with the +new definition, `INSERT INTO "__new_
" (...) SELECT ...` for every column both tables +have, `DROP TABLE`, `ALTER TABLE "__new_
" RENAME TO "
"`, `PRAGMA foreign_keys=ON`, +then every index of the table again. A copy fails when the rows do not fit the new table, such +as a column made NOT NULL over existing NULLs or a new NOT NULL column with no default. A +column a rebuild leaves out needs the same `confirm_data_loss` hint as a dropped column. + +A rename is never guessed. Without a hint it reads as a drop plus an add, and the drop asks for +confirmation. Pass the rename instead: + +```json +[ + { "type": "confirm_rename", "kind": "table", "entity": "links", "to": "urls" }, + { "type": "confirm_rename", "kind": "column", "entity": "urls.url", "to": "target" } +] +``` + +`entity` is the old table name, or `
.` with the table's new name. Table +renames run first. Indexes keep their names and follow the rename. + +### Adopting a drizzle-kit SQLite folder + +`effect-orm generate --baseline --from-drizzle` reads drizzle-kit 1.0's last `snapshot.json` +(version 7, `dialect: "sqlite"`): tables, columns, primary keys, indexes and UNIQUE +constraints. Foreign keys, checks, views, generated and AUTOINCREMENT columns and partial +indexes are listed, and nothing is written. The baseline runs nothing. A later `generate` +compares your `SQLite.table` definitions with it; when they declare the same tables, it writes +nothing. The tests prove this with a folder drizzle-kit 1.0.0-rc.5 generated. + +drizzle-kit 1.0 leaves `NOT NULL` out of a primary key column (`"id" text PRIMARY KEY`), so +SQLite then accepts a NULL key in a `text` key column. Its snapshot marks the column +`notNull: false`, but drizzle-kit treats a key column as not null. The baseline reads it as not +null too, so a `SQLite.table` key column (always `NOT NULL`) matches it and no rebuild is +written. The database keeps accepting a NULL key until that table is rebuilt for another change. + +The drizzle-kit 0.x layout (`meta/_journal.json` and flat `.sql` files) is not read. Run +`drizzle-kit up` to convert it first. + +`/migrate` does not run SQLite folders yet: `run`, `status`, `verify` and `baseline` fail with a +`MigrateSourceError` for one. Keep applying them with the tool you use now. + diff --git a/docs/reference.md b/docs/reference.md index abc2d46..d948955 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -5,9 +5,10 @@ The package has one entry per database, each self-contained: ```ts import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" +import * as SQLite from "@maple-dev/effect-orm/sqlite" ``` -Both carry the whole [query builder](#query-builder-both-entries). Each adds its own column +All carry the whole [query builder](#query-builder-every-entry). Each adds its own column types, functions, table definitions and a `compile` that defaults to its dialect: [`/clickhouse`](#clickhouse) and [`/postgres`](#postgres). Everything else is on a [subpath](#other-subpaths) for a narrower job. @@ -32,9 +33,9 @@ uniformly, which some codebases prefer for exactly this reason. --- -## Query builder (both entries) +## Query builder (every entry) -Everything in this section is exported, identically, from both `/clickhouse` and `/postgres`. +Everything in this section is exported, identically, from `/clickhouse`, `/postgres` and `/sqlite`. ### Query construction @@ -46,7 +47,7 @@ Everything in this section is exported, identically, from both `/clickhouse` and | `unionAll` | `(...queries) => CHUnionQuery` | | `update` | `(table) => CHUpdateStart`, then `CHUpdate`: `.set(record \| fn)`, `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)`. See [Updating and deleting](./updates-and-deletes.md) | | `deleteFrom` | `(table) => CHDelete`: `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)` | -| `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres). See [Inserting rows](./inserts.md) | +| `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres and SQLite). See [Inserting rows](./inserts.md) | Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or [`PG.table`](#tables-and-ddl-1). @@ -240,7 +241,7 @@ Tag `"@maple-dev/effect-orm/CompiledQueryDecodeError"`. Fails the `decodeRows` / #### `SchemaDefinitionError` -Tag `"@maple-dev/effect-orm/SchemaDefinitionError"`. Fails `S.entitiesOf`, `S.pgEntitiesOf` and +Tag `"@maple-dev/effect-orm/SchemaDefinitionError"`. Fails `S.entitiesOf`, `S.pgEntitiesOf`, `S.sqliteEntitiesOf` and `TB.buildProject` when a definition cannot render: a name that is not a plain identifier, a MergeTree-family engine with no `orderBy`, a view reading from a union, a view writing to a table outside the schema. Fields: `problems` (every `DefinitionProblem`, `{ object, message }`) and @@ -509,7 +510,7 @@ placeholders, double-quoted identifiers. | `SqlQuery`, `compileQuery` | `/sql` | | `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | | `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | -| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint` | `/schema` | +| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint`; for SQLite, `sqliteEntitiesOf`, `renderSqliteSchema`, `renderSqliteCreateTable`, `renderSqliteCreateIndex`, `diffSqliteSchemas`, `fromDrizzleSqliteSnapshot`, `SqliteMigrationOp`, `renderSqliteOp`, `renderSqliteMigrationSql`, `labelOfSqlite`, `rebuildName`, the `SqliteSnapshot` schema and the `SqliteTableEntity`, `SqliteColumnEntity`, `SqliteIndexEntity`, `SqliteUniqueEntity` schemas | `/schema` | | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | @@ -521,8 +522,8 @@ placeholders, double-quoted identifiers. | `makeHttpClient`, `makeHttpTransport`, `httpConfigFromEnv` | `/benchmark/http` | | `runCli` | `/benchmark/cli` | -`/schema` reads tables; it does not declare them. Declare tables with `table` from `/clickhouse` -or `/postgres`. See [Schema and migrations](./migrations.md), [Database](./database.md), +`/schema` reads tables; it does not declare them. Declare tables with `table` from `/clickhouse`, +`/postgres` or `/sqlite`. See [Schema and migrations](./migrations.md), [Database](./database.md), [Running a query](./running-queries.md) for what the `/sql` statement helpers are for, and [Benchmarking](./benchmarking.md) for the complete benchmark API, command workflow, result verification, and JSON protocol. @@ -530,3 +531,67 @@ verification, and JSON protocol. Note `/sql` exports a `compile` (fragment → string) distinct from the dialect entries' `compile` (query → `CompiledQuery`), and a `when` distinct from the query builder's `when` (optional conditions). + +## SQLite and D1 + +`@maple-dev/effect-orm/sqlite` exports the shared query builder and the four compile functions. +`sqliteDialect` is the default; `d1Dialect` uses the same SQL with a 100-binding limit and no +interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and limits. + +| Export | Purpose | +| --- | --- | +| `SqliteType` | A SQLite SQL type with its Effect codec | +| `text`, `integer`, `real` | Text, safe integer and finite number codecs | +| `boolean` | A boolean encoded as 0/1; other integers fail decoding | +| `timestamp` | An instant stored as ISO text | +| `blob` | Bytes from a Uint8Array or D1's array wire representation | +| `json(schema)` | JSON text decoded through the supplied schema | +| `nullable(type)`, `custom`, `brand` | Type composition | +| `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min`, `max` | SQLite aggregates; the last four can return SQL NULL | +| `lower`, `upper`, `length`, `coalesce` | SQLite scalar functions | + +### Tables and DDL + +```ts +const Links = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.column(SQLite.text, { unique: true }), + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + clickCount: SQLite.column(SQLite.integer, { name: "click_count", default: 0 }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["id"], + indexes: [SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) +``` + +| Export | Signature | +| --- | --- | +| `table` | `(name, TableDefinition) => SqliteSchemaTable`, `(name, ExternalTableDefinition) => Table`, or `(name, columns, TableOptions?) => Table`, a query schema with no DDL | +| `column` | `(type, ColumnOptions) => ColumnSpec` | +| `index` | `(name, on) => IndexSpec`: columns, or a callback building expressions | +| `uniqueIndex` | The same, `UNIQUE` | + +**`TableDefinition`** — `columns` (required), `primaryKey` (column names: one is written inline, +several as a table constraint), `indexes`, `tenantColumn`. A column is `NOT NULL` unless its +type is `nullable(...)`; a primary key column cannot be nullable. + +**`ExternalTableDefinition`** — `{ external: true, columns, tenantColumn? }`: a view, or a table +another tool migrates. No DDL. **`TableOptions`** — `tenantColumn`, `defaults`, `computed`, for +the bare-columns form. + +**`ColumnOptions`** — one of `default` (a literal, encoded through the column type; a boolean +is written `true` / `false`) or `defaultExpr` (SQL as written, so add SQLite's parentheses, or a +DSL callback, parenthesized for you); either lets an insert leave the column out +(`DefaultedColumnsOf`). Also `name` (the column's name in SQL) and `unique` (`true` or a +constraint name), an inline `UNIQUE` constraint named `
__unique` by default, as +drizzle-kit 1.0 writes and names it. + +**Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `TableDdl`, `SqliteSchemaTable`, +`DdlExpr`, `DdlKey`, `DefinitionProblem`. A definition that cannot render records a problem; +`S.sqliteEntitiesOf` fails with a `SchemaDefinitionError`, and `S.renderSqliteSchema` writes +the `CREATE TABLE` and `CREATE INDEX` statements. `effect-orm generate` with `dialect: "sqlite"` +writes migrations for them; see [SQLite migrations](./migrations.md#sqlite). The `/migrate` +runner does not apply SQLite migrations yet. diff --git a/docs/sqlite.md b/docs/sqlite.md new file mode 100644 index 0000000..b6ce4d1 --- /dev/null +++ b/docs/sqlite.md @@ -0,0 +1,177 @@ +# SQLite and Cloudflare D1 + +Import `@maple-dev/effect-orm/sqlite` for the shared query builder, SQLite codecs and functions. +Compilation uses positional `?` bindings. Each occurrence of a parameter gets its own binding. +The package opens no connection: use the Effect `SqlClient` you already provide. + +## Define and compile a query + +```ts title="sqlite-quickstart.ts" +import * as SQLite from "@maple-dev/effect-orm/sqlite" + +export const Acks = SQLite.table("acknowledgments", { + id: SQLite.text, + org_id: SQLite.text, + entry_id: SQLite.text, +}, { tenantColumn: "org_id" }) + +export const byOrg = SQLite.from(Acks) + .select("entry_id") + .where(($) => [$.org_id.eq(SQLite.param.string("orgId"))]) + .orderBy(["entry_id", "asc"]) + +export const compiled = SQLite.compileUnsafe(byOrg, { orgId: "org_1" }) +// compiled.parameters is ["org_1"]; compiled.tenantScope is "single-tenant". +``` + +`table(name, columns, options?)` is a query schema only: it carries no DDL. + +## Declare a table with its DDL + +Pass `{ columns, primaryKey, indexes }` to `table` to declare the table's DDL too. The value is +still a query `Table`. `sqliteEntitiesOf` from `/schema` checks a set of tables as one schema, +and `renderSqliteSchema` writes their `CREATE TABLE` and `CREATE INDEX` statements: + +```ts title="sqlite-ddl.ts" +import { Effect } from "effect" +import * as S from "@maple-dev/effect-orm/schema" +import * as SQLite from "@maple-dev/effect-orm/sqlite" + +export const ShortLinks = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.text, + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + clickCount: SQLite.column(SQLite.integer, { name: "click_count", default: 0 }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) + +export const statements = S.sqliteEntitiesOf([ShortLinks]).pipe(Effect.map(S.renderSqliteSchema)) +// CREATE TABLE IF NOT EXISTS "short_link" ( +// "id" text PRIMARY KEY NOT NULL, +// "code" text NOT NULL, +// "org_id" text, +// "click_count" integer NOT NULL DEFAULT 0, +// "created_at" text NOT NULL DEFAULT (CURRENT_TIMESTAMP) +// ) +// CREATE UNIQUE INDEX IF NOT EXISTS "short_link_code" ON "short_link" ("code") +// CREATE INDEX IF NOT EXISTS "short_link_org" ON "short_link" ("org_id") +``` + +- A column is `NOT NULL` unless its type is `nullable(...)`. A primary key column cannot be + nullable: SQLite would accept NULL keys in it. +- One primary key column is written inline (`PRIMARY KEY`); several become a table constraint + `PRIMARY KEY ("a", "b")`. SQLite stores no constraint name, so there is none to give. +- `default` takes a literal of the column type. `defaultExpr` takes SQL as written, so add the + parentheses SQLite requires around an expression, or a DSL callback, which is parenthesized + for you. Either lets an insert leave the column out. +- `unique: true` writes an inline `UNIQUE` constraint, as drizzle-kit 1.0 does for `.unique()`. + Its name, `
__unique` or the string you give, is kept in the schema entities; + SQLite itself names the backing index `sqlite_autoindex_...`. +- Column types declare their SQLite type: `text`, `timestamp` and `json` are `text`; `integer` + and `boolean` are `integer`; `real` is `real`; `blob` is `blob`. A `custom(sql, schema)` type + declares `sql`, for example `numeric`. +- `table(name, { external: true, columns })` declares a table another tool owns: no DDL. + +The tests apply this DDL and drizzle-kit's SQL for the same tables to real SQLite, and compare +`PRAGMA table_info`, `index_list` and `index_xinfo`. One difference is deliberate: drizzle-kit +1.0 drops `NOT NULL` from a primary key column, so SQLite then accepts a NULL key in a `text` +key column. This entry keeps `NOT NULL`. + +`effect-orm generate` with `dialect: "sqlite"` diffs these tables against the migrations folder +and writes the next migration, in the drizzle-kit 1.0 folder layout. See +[SQLite migrations](./migrations.md#sqlite). effect-orm does not apply SQLite migrations yet, +and has no SQLite introspection or drift check: apply the folder with a tool that reads +drizzle-kit folders, such as Alchemy's D1 resource. `WITHOUT ROWID`, +`STRICT`, `AUTOINCREMENT`, partial indexes, foreign keys and `CHECK` constraints are not +declared yet. + +The entry exports `text`, `integer` (a safe JavaScript integer), `real` (finite numbers), +`boolean` (0/1), `blob`, `timestamp` (ISO text), `json(schema)`, `nullable`, `custom` and `brand`. +`blob` accepts the Uint8Array wire form from Node and the array-of-bytes form from D1. +`json` always uses the supplied schema to validate decoded JSON. +Use `text` when a stored date/time must remain a string. + +SQLite aggregates include `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min` and `max`. +The last four return NULL over an empty input. Scalar helpers include `lower`, `upper`, +`length` and `coalesce`. SQLite's own text and case rules apply. Foreign built-in function +sets, such as ClickHouse's `count()`, fail compilation instead of producing invalid SQL. + +## Execute on SQLite + +Install a compatible Effect SQL driver, for example `@effect/sql-sqlite-node` on Node 24+. +No driver is a runtime dependency of this package. Use current SQLite with RETURNING support. + +```ts title="sqlite-node.ts" +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import { sqliteDialect } from "@maple-dev/effect-orm/sqlite" + +export const SQLiteLive = Db.layerSqlClient({ + dialect: sqliteDialect, + requireTenantScope: true, +}).pipe(Layer.provide(SqliteClient.layer({ filename: ":memory:" }))) +``` + +`Db.run(query, params)` compiles for the database dialect and decodes the selected rows. +SQLite transactions and nested savepoints use the supplied client's `withTransaction`. +`serializable` is the only accepted explicit isolation level. Per-transaction access modes +and deferrable settings are refused. The driver must support the declared transaction behavior; +use the D1 profile for a D1 client. + +## Execute on D1 + +Install `@effect/sql-d1` with a compatible Effect version. Provide the Worker's D1 binding: + +```ts title="sqlite-d-one.ts" +import * as D1Client from "@effect/sql-d1/D1Client" +import { Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import { d1Dialect } from "@maple-dev/effect-orm/sqlite" + +export const d1DatabaseLayer = (db: D1Client.D1ClientConfig["db"]) => + Db.layerSqlClient({ dialect: d1Dialect, requireTenantScope: true }).pipe( + Layer.provide(D1Client.layer({ db })), + ) +``` + +Use `d1Dialect` when compiling a query separately as well: +`SQLite.compile(query, params, { dialect: SQLite.d1Dialect })`. A query compiled for `sqlite` +cannot later run on a database configured as `d1`; pass the builder to `Db.run` instead. + +D1 compilation refuses more than [100 bound parameters per statement](https://developers.cloudflare.com/d1/platform/limits/). +It never splits a write implicitly. Chunk an import explicitly if needed; several chunks do +not make one atomic write. SQLite compilation uses its modern default limit of 32766; a driver +with a lower limit needs a custom dialect profile. + +`Database.transaction` on D1 returns `TransactionUnsupported` before executing its body. +D1's [atomic batch API](https://developers.cloudflare.com/d1/worker-api/d1-database/#batch) +executes a fixed list of statements. It cannot run an arbitrary Effect transaction body with +reads that determine later writes. There is no ORM batch API in this first implementation. + +## Safety and syntax limits + +Derived schemas validate returned rows. A row with the wrong type returns +`CompiledQueryDecodeError`. Input values use each column's encoder before binding. + +Tenant enforcement is **opt-in**, through `requireTenantScope: true`. A compiled query over a +tenant table must pin its `tenantColumn` to one value, or explicitly use `.crossTenant()` for +an administrative read/write. Raw SQL passed to `query` or `execute` has no tenant proof and +bypasses this guard. It is not a replacement for an application's authorization checks. + +The dialect handles OFFSET without LIMIT, branch-local ordering/limits in UNION ALL, and +INSERT SELECT with ON CONFLICT. It refuses these unsupported forms at compile time: + +- `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE` with SQLite's own case rules. +- `ON CONSTRAINT` conflict targets. Name the indexed columns instead. +- Per-cell `DEFAULT` in a VALUES tuple. Omit a defaulted column from every row in the statement, + or group rows by their present columns. All-default rows are not supported by the builder. +- ClickHouse FORMAT/SETTINGS and built-in functions from another dialect. + +The tests run real SQLite through `@effect/sql-sqlite-node` and local workerd/D1 through +Miniflare. They require no deployed database or Cloudflare credentials. diff --git a/package.json b/package.json index 079ebcd..5238372 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@maple-dev/effect-orm", "version": "0.7.2", - "description": "Type-safe ClickHouse and Postgres queries, result decoding, and reproducible benchmarks for Effect and TypeScript", + "description": "Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproducible benchmarks for Effect and TypeScript", "keywords": [ "clickhouse", "effect", @@ -9,7 +9,9 @@ "postgres", "query-builder", "sql", - "typescript" + "typescript", + "sqlite", + "cloudflare-d1" ], "homepage": "https://effect-clickhouse.maple.dev", "bugs": { @@ -97,6 +99,11 @@ "types": "./dist/benchmark/cli.d.mts", "import": "./dist/benchmark/cli.mjs", "default": "./dist/benchmark/cli.mjs" + }, + "./sqlite": { + "types": "./dist/sqlite.d.mts", + "import": "./dist/sqlite.mjs", + "default": "./dist/sqlite.mjs" } }, "publishConfig": { @@ -121,13 +128,16 @@ "@effect/language-service": "^0.87.3", "@effect/platform-bun": "4.0.0", "@effect/sql-clickhouse": "4.0.0", + "@effect/sql-d1": "4.0.0", "@effect/sql-pglite": "4.0.0", + "@effect/sql-sqlite-node": "4.0.0", "@effect/vitest": "4.0.0", "@electric-sql/pglite": "0.5.8", "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", "fast-check": "^4.9.0", + "miniflare": "4.20260730.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9" diff --git a/scripts/check-exports-documented.mjs b/scripts/check-exports-documented.mjs index 93bec71..0aa5ff8 100644 --- a/scripts/check-exports-documented.mjs +++ b/scripts/check-exports-documented.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Every export of the two dialect entries is named in docs/reference.md, whose +// Every export of the three dialect entries is named in docs/reference.md, whose // first line claims to be the catalog. A claim like that decays the moment // anything is added, so it is checked rather than promised. // @@ -11,7 +11,7 @@ import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" const root = join(dirname(fileURLToPath(import.meta.url)), "..") -const entries = ["clickhouse", "postgres"] +const entries = ["clickhouse", "postgres", "sqlite"] const exportsOf = (entry) => { const dts = join(root, `dist/${entry}.d.mts`) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..cb2a4e5 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -936,7 +936,9 @@ function compileInner< const sqlQuery: SqlQuery = { distinct: state.distinct !== undefined, distinctOn: Array.isArray(state.distinct) - ? (state.distinct.length === 0 + ? (currentDialect().clauses.distinctOn === false + ? fail>(new QueryBuilderDefect({ message: `${currentDialect().name} has no DISTINCT ON clause` }), []) + : state.distinct.length === 0 ? fail>(new QueryBuilderDefect({ message: "CHQuery: distinctOn() needs at least one key" }), []) : state.distinct ).map((key: string) => { @@ -967,7 +969,9 @@ function compileInner< orderBy: state.orderByFn === undefined ? orderByClause(state.orderBySpecs).map(raw) : orderByExprs(state.orderByFn($)).map(raw), - limit: rowCount("limit", state.limitValue), + limit: state.limitValue == null && state.offsetValue != null && currentDialect().clauses.offsetRequiresLimit === true + ? raw("-1") + : rowCount("limit", state.limitValue), offset: rowCount("offset", state.offsetValue), format: options?.skipFormat ? undefined : formatClause(state.formatValue), } @@ -1442,7 +1446,9 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => currentDialect().clauses.wrapUnionBranches === true + ? `SELECT * FROM (\n${q.sql}\n) AS ${quoteIdent("__union_branch")}` + : parenthesize ? `(\n${q.sql}\n)` : q.sql).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = @@ -1455,6 +1461,8 @@ function compileUnionInner, Params extends Re } if (state.outerLimitValue != null) { sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` + } else if (state.outerOffsetValue != null && currentDialect().clauses.offsetRequiresLimit === true) { + sql += "\nLIMIT -1" } if (state.outerOffsetValue != null) { sql += `\nOFFSET ${compileSqlFragment(rowCount("offset", state.outerOffsetValue)!)}` @@ -1684,6 +1692,9 @@ const onConflictClause = ( let target = "" if (conflict.target !== undefined) { if ("constraint" in conflict.target) { + if (dialect.clauses.onConflictConstraint === false) { + return fail(new QueryBuilderDefect({ message: `${where}: ${dialect.name} has no ON CONSTRAINT conflict target; name its columns instead` }), "") + } if (conflict.targetWhere !== undefined) { return fail(new QueryBuilderDefect({ message: `${where}: targetWhere needs a column target, not a constraint` }), "") } @@ -1778,7 +1789,9 @@ const valueCells = (table: Table, params: Record = { ...params } let next = 0 const cell = (column: string, value: unknown, context: string): string => { - if (value === undefined) return "DEFAULT" + if (value === undefined) return currentDialect().clauses.defaultInValues === false + ? fail(new QueryBuilderDefect({ message: `${where}: ${currentDialect().name} has no per-cell DEFAULT in VALUES; group rows by their present columns` }), "NULL") + : "DEFAULT" if (isExprLike(value)) return compileSqlFragment(value.toFragment()) const wire = encodeValue(table.columns[column]!.literalSchema, value, `${where}: ${context}, column ${column}`) let name = `${VALUE_PARAM}${next++}` @@ -1954,7 +1967,9 @@ function compileInsert(insert: CHInsert, params: Record ({ dialect: dialect.name, + ilike: dialect.ilike, quoteIdent: dialect.quoteIdent, quoteString: (value) => checked(dialect, dialect.quoteString(value), "a string literal"), literal: (value, context) => checkedLiteral(dialect, value, context), diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 99773df..34bb9f1 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -13,7 +13,7 @@ import { activeSqlSyntax } from "../sql/sql-syntax" import { activeLiteralBinder } from "../sql/literal-binder" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferEncoded, type InferTS } from "./types" import { encodeColumnLiteral, isSubnormal, numberText, sqlLiteral } from "./literal" -import { QueryBuilderError } from "./errors" +import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { fail } from "./failure" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" @@ -533,7 +533,9 @@ export function makeExpr( like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}`)), notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}`)), - ilike: (pattern) => makeCond(known(() => `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), + ilike: (pattern) => makeCond(known(() => activeSqlSyntax()?.ilike === false + ? fail(new QueryBuilderDefect({ message: `ILIKE is not supported by the ${activeSqlSyntax()?.dialect} dialect; use LIKE with SQLite's own case rules` }), "FALSE") + : `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))), notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))), diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 766d2a5..7e63f68 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -21,10 +21,11 @@ import { QueryBuilderDefect } from "../errors" import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "../expr" import type { FnResult } from "../define-fn" -export type FunctionSet = "clickhouse" | "postgres" +export type FunctionSet = "clickhouse" | "postgres" | "sqlite" export type BuiltinKind = "scalar" | "aggregate" | "window" const setLabel: Record = { + sqlite: "a SQLite function (from @maple-dev/effect-orm/sqlite)", clickhouse: "a ClickHouse function (from @maple-dev/effect-orm/clickhouse)", postgres: "a Postgres function (from @maple-dev/effect-orm/postgres)", } diff --git a/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql new file mode 100644 index 0000000..b105d8e --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/migration.sql @@ -0,0 +1,34 @@ +CREATE TABLE `membership` ( + `org_id` text NOT NULL, + `user_id` text NOT NULL, + `role` text DEFAULT 'member' NOT NULL, + `weight` real DEFAULT 1.5 NOT NULL, + `active` integer DEFAULT true NOT NULL, + `avatar` blob, + `amount` numeric, + `email` text UNIQUE, + `created_at` text DEFAULT (CURRENT_TIMESTAMP) NOT NULL, + CONSTRAINT `membership_pk` PRIMARY KEY(`org_id`, `user_id`) +); +--> statement-breakpoint +CREATE TABLE `short_link` ( + `id` text PRIMARY KEY, + `code` text NOT NULL, + `target_url` text NOT NULL, + `target_hash` text NOT NULL, + `org_id` text, + `created_by` text, + `subject_type` text NOT NULL, + `subject_id` text NOT NULL, + `purpose` text NOT NULL, + `prospect_id` text, + `click_count` integer NOT NULL, + `first_accessed_at` text, + `last_accessed_at` text, + `created_at` text NOT NULL, + `updated_at` text NOT NULL +); +--> statement-breakpoint +CREATE INDEX `membership_user_role` ON `membership` (`user_id`,`role`);--> statement-breakpoint +CREATE UNIQUE INDEX `short_link_code` ON `short_link` (`code`);--> statement-breakpoint +CREATE INDEX `short_link_org` ON `short_link` (`org_id`); \ No newline at end of file diff --git a/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json new file mode 100644 index 0000000..1877350 --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/20261025075810_init/snapshot.json @@ -0,0 +1,333 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "815592fa-041d-4bbb-b8c4-a994b30babe5", + "prevIds": [ + "00000000-0000-0000-0000-000000000000" + ], + "ddl": [ + { + "name": "membership", + "entityType": "tables" + }, + { + "name": "short_link", + "entityType": "tables" + }, + { + "table": "membership", + "name": "org_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "user_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "role", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'member'", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "weight", + "type": "real", + "notNull": true, + "autoincrement": false, + "default": "1.5", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "active", + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "avatar", + "type": "blob", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "amount", + "type": "numeric", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "email", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "created_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "(CURRENT_TIMESTAMP)", + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "code", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "target_url", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "target_hash", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "org_id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "created_by", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "subject_type", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "subject_id", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "purpose", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "prospect_id", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "click_count", + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "first_accessed_at", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "last_accessed_at", + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "created_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "short_link", + "name": "updated_at", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "entityType": "columns" + }, + { + "table": "membership", + "name": "membership_user_role", + "columns": [ + { + "value": "user_id", + "isExpression": false + }, + { + "value": "role", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "short_link", + "name": "short_link_code", + "columns": [ + { + "value": "code", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "short_link", + "name": "short_link_org", + "columns": [ + { + "value": "org_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "entityType": "indexes" + }, + { + "table": "membership", + "name": "membership_pk", + "columns": [ + "org_id", + "user_id" + ], + "nameExplicit": false, + "entityType": "pks" + }, + { + "table": "short_link", + "name": "short_link_pk", + "columns": [ + "id" + ], + "nameExplicit": false, + "entityType": "pks" + }, + { + "table": "membership", + "name": "membership_email_unique", + "columns": [ + "email" + ], + "nameExplicit": false, + "entityType": "uniques" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt b/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt new file mode 100644 index 0000000..e886b25 --- /dev/null +++ b/src/kit/__fixtures__/drizzle-sqlite/schema.ts.txt @@ -0,0 +1,19 @@ +// The drizzle-orm 1.0.0-rc.5 schema drizzle-kit 1.0.0-rc.5 `generate --name init` read to write this folder. +import { sqliteTable, text, integer, real, blob, numeric, index, uniqueIndex, primaryKey } from "drizzle-orm/sqlite-core" +import { sql } from "drizzle-orm" +export const shortLink = sqliteTable("short_link", { + id: text("id").primaryKey().notNull(), code: text("code").notNull(), targetUrl: text("target_url").notNull(), + targetHash: text("target_hash").notNull(), orgId: text("org_id"), createdBy: text("created_by"), + subjectType: text("subject_type").notNull(), subjectId: text("subject_id").notNull(), purpose: text("purpose").notNull(), + prospectId: text("prospect_id"), clickCount: integer("click_count").notNull(), + firstAccessedAt: text("first_accessed_at"), lastAccessedAt: text("last_accessed_at"), + createdAt: text("created_at").notNull(), updatedAt: text("updated_at").notNull(), +}, (t) => [uniqueIndex("short_link_code").on(t.code), index("short_link_org").on(t.orgId)]) +export const membership = sqliteTable("membership", { + orgId: text("org_id").notNull(), userId: text("user_id").notNull(), + role: text("role").notNull().default("member"), weight: real("weight").notNull().default(1.5), + active: integer("active", { mode: "boolean" }).notNull().default(true), + avatar: blob("avatar"), amount: numeric("amount"), + email: text("email").unique(), + createdAt: text("created_at").notNull().default(sql`(CURRENT_TIMESTAMP)`), +}, (t) => [primaryKey({ columns: [t.orgId, t.userId] }), index("membership_user_role").on(t.userId, t.role)]) diff --git a/src/kit/cli.ts b/src/kit/cli.ts index 9cbf026..3b7cdec 100644 --- a/src/kit/cli.ts +++ b/src/kit/cli.ts @@ -9,7 +9,7 @@ import { Hints, type Hint } from "../schema/diff" import { check, generate, KitError, readMigrations, type KitConfig } from "./generate" import { pull } from "./pull" -const help = `effect-orm: schema migrations for ClickHouse and Postgres +const help = `effect-orm: schema migrations for ClickHouse and Postgres, and generated SQLite migrations generate [--name x] [--custom] Diff the schema against the migrations folder and write the next migration [--hints ] [--hints-file ] @@ -19,7 +19,7 @@ const help = `effect-orm: schema migrations for ClickHouse and Postgres Write table definitions from the database (ClickHouse). Default --out: the config's schema module, when it names one. Then run generate --baseline check Validate the migrations folder (snapshot chain, branch conflicts) - migrate [--strict] Apply pending migrations (needs config.driver) + migrate [--strict] Apply pending migrations (needs config.driver; not SQLite yet) status Applied, pending, partial, or changed, per migration verify Compare the database with the last applied snapshot baseline Record and every one before it as applied, without running them @@ -68,7 +68,9 @@ const promptConfirm = (hint: Hint): Effect.Effect => const rl = createInterface({ input: process.stdin, output: process.stdout }) const answer = await rl.question( hint.type === "confirm_rename" - ? `Rename enum label ${hint.entity} to ${JSON.stringify(hint.to)}? Rows holding it will hold the new label. [y/N] ` + ? hint.kind === "enum_value" + ? `Rename enum label ${hint.entity} to ${JSON.stringify(hint.to)}? Rows holding it will hold the new label. [y/N] ` + : `Rename ${hint.kind} ${hint.entity} to ${hint.to}? [y/N] ` : `Drop ${hint.kind} ${hint.entity}? ${hint.kind === "enum" ? "Postgres refuses while a column uses it." : "Its data will be lost."} [y/N] `, ) rl.close() diff --git a/src/kit/generate.ts b/src/kit/generate.ts index 8d88bbb..c1d6f8e 100644 --- a/src/kit/generate.ts +++ b/src/kit/generate.ts @@ -10,6 +10,7 @@ import { fromRecord, isStagingName, STATEMENT_BREAKPOINT, type LoadedMigration, import type { MigrationDriver } from "../migrate/driver" import { diffSchemas, type DiffResult, type Hint } from "../schema/diff" import { fromDrizzleSnapshot } from "../schema/drizzle" +import { fromDrizzleSqliteSnapshot } from "../schema/sqlite-drizzle" import { ORIGIN_ID, type AnySchemaEntity, type SchemaDialect, type SchemaEntity } from "../schema/entities" import type { SchemaDefinitionError } from "../schema/problems" import { labelOf, renderOp, type MigrationFile, type MigrationOp } from "../schema/ops" @@ -17,6 +18,10 @@ import { diffPgSchemas } from "../schema/pg-diff" import type { PgSchemaEntity } from "../schema/pg-entities" import { labelOfPg, renderPgOp, type PgMigrationOp } from "../schema/pg-ops" import type { RenderOptions } from "../schema/render" +import { sqliteEntitiesOf, type SqliteSchemaTable } from "../schema/sqlite-define" +import { diffSqliteSchemas } from "../schema/sqlite-diff" +import type { SqliteSchemaEntity } from "../schema/sqlite-entities" +import { labelOfSqlite, renderSqliteMigrationSql, renderSqliteOp, type SqliteMigrationOp } from "../schema/sqlite-ops" import { dialectOfObject, entitiesOf, @@ -31,7 +36,7 @@ import { analyze, type GraphProblem } from "./graph" export interface KitConfig { /** The database this folder migrates. Default `clickhouse`. */ readonly dialect?: SchemaDialect - /** Modules whose exports include `table` / `materializedView` values from `/clickhouse` or `/postgres`. */ + /** Modules whose exports include `table` / `materializedView` values from `/clickhouse`, `/postgres` or `/sqlite`. */ readonly schema: string | ReadonlyArray /** The migrations folder. */ readonly out: string @@ -43,6 +48,7 @@ export interface KitConfig { * What a generated Postgres migration holds. `ops` (default) writes `migration.json`; * `sql` writes the rendered statements to `migration.sql`, split by * `--> statement-breakpoint`, so a tool that applies drizzle-kit folders applies it too. + * A SQLite migration is always `migration.sql`. */ readonly emit?: "ops" | "sql" } @@ -193,13 +199,16 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption } let file: MigrationFile | undefined + /** A SQLite migration is SQL: there is no ops file for it. */ + let sqliteOps: ReadonlyArray | undefined let snapshotEntities: ReadonlyArray let prevIds = analysis.baseIds if (options.baseline === "drizzle") { - if (dialect !== "postgres") return yield* new KitError({ code: "config", message: "--from-drizzle needs dialect: \"postgres\"" }) + if (dialect === "clickhouse") return yield* new KitError({ code: "config", message: "--from-drizzle needs dialect: \"postgres\" or \"sqlite\"" }) const source = [...analysis.legacy].reverse().find((m) => m.foreignSnapshot?.tool === "drizzle-kit") if (source === undefined) return yield* new KitError({ code: "config", message: "no drizzle-kit snapshot.json in the folder to start from" }) - const imported = fromDrizzleSnapshot(source.foreignSnapshot!.json) + const imported = + dialect === "sqlite" ? fromDrizzleSqliteSnapshot(source.foreignSnapshot!.json) : fromDrizzleSnapshot(source.foreignSnapshot!.json) if (imported.unsupported.length > 0) { return yield* new KitError({ code: "unsupported", @@ -212,7 +221,11 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption } else { const objects = yield* loadSchema(config, cwd) const entities: Effect.Effect, SchemaDefinitionError> = - dialect === "postgres" ? pgEntitiesOf(objects) : entitiesOf(objects) + dialect === "postgres" + ? pgEntitiesOf(objects) + : dialect === "sqlite" + ? sqliteEntitiesOf(objects as ReadonlyArray>) + : entitiesOf(objects) const next = yield* entities.pipe( Effect.mapError( (error) => @@ -228,10 +241,12 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption else if (options.custom === true) snapshotEntities = analysis.base else { const hints = [...(options.hints ?? [])] - const diff = (): DiffResult => + const diff = (): DiffResult => dialect === "postgres" ? diffPgSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) - : diffSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) + : dialect === "sqlite" + ? diffSqliteSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) + : diffSchemas(analysis.base as ReadonlyArray, next as ReadonlyArray, hints) let result = diff() if (result.unsupported.length > 0) { return yield* new KitError({ @@ -254,10 +269,9 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption }) } if (result.ops.length === 0) return { written: undefined, plan: [] } satisfies GenerateResult - file = - dialect === "postgres" - ? { version: "1", dialect: "postgres", ops: result.ops as ReadonlyArray } - : { version: "1", ops: result.ops as ReadonlyArray } + if (dialect === "sqlite") sqliteOps = result.ops as ReadonlyArray + else if (dialect === "postgres") file = { version: "1", dialect: "postgres", ops: result.ops as ReadonlyArray } + else file = { version: "1", ops: result.ops as ReadonlyArray } } } @@ -269,29 +283,31 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption // Written aside and renamed into place, so a reader never sees half a migration. yield* Effect.gen(function* () { yield* io(() => mkdir(staging, { recursive: true }), `Cannot create ${staging}`) - yield* io( - () => - file === undefined - ? writeFile( - join(staging, "migration.sql"), + const [fileName, contents] = + sqliteOps !== undefined + ? ["migration.sql", renderSqliteMigrationSql(sqliteOps, STATEMENT_BREAKPOINT)] + : file === undefined + ? [ + "migration.sql", options.baseline !== undefined ? "-- Baseline: the schema as it stood when effect-orm took over this folder. Runs nothing.\n" : "-- Custom SQL migration. Separate statements with a line holding only:\n-- --> statement-breakpoint\n", - ) + ] : "dialect" in file && config.emit === "sql" - ? writeFile(join(staging, "migration.sql"), renderPgSqlFile(file.ops)) - : writeFile(join(staging, "migration.json"), `${JSON.stringify(file, null, "\t")}\n`), - `Cannot write ${dir}`, - ) + ? ["migration.sql", renderPgSqlFile(file.ops)] + : ["migration.json", `${JSON.stringify(file, null, "\t")}\n`] + yield* io(() => writeFile(join(staging, fileName), contents), `Cannot write ${dir}`) yield* io(() => writeFile(join(staging, "snapshot.json"), serializeSnapshot(snapshot)), `Cannot write ${dir}`) yield* io(() => rename(staging, dir), `Cannot move ${staging} to ${dir}`) }).pipe(Effect.onError(() => Effect.promise(() => rm(staging, { recursive: true, force: true }).catch(() => undefined)))) const plan = - file === undefined - ? [] - : "dialect" in file - ? file.ops.map((op) => ({ label: labelOfPg(op), sql: renderPgOp(op) })) - : file.ops.map((op) => ({ label: labelOf(op), sql: renderOp(op, config.render) })) + sqliteOps !== undefined + ? sqliteOps.map((op) => ({ label: labelOfSqlite(op), sql: renderSqliteOp(op) })) + : file === undefined + ? [] + : "dialect" in file + ? file.ops.map((op) => ({ label: labelOfPg(op), sql: renderPgOp(op) })) + : file.ops.map((op) => ({ label: labelOf(op), sql: renderOp(op, config.render) })) return { written: dir, plan } satisfies GenerateResult }) diff --git a/src/kit/sqlite-generate.test.ts b/src/kit/sqlite-generate.test.ts new file mode 100644 index 0000000..e395a3d --- /dev/null +++ b/src/kit/sqlite-generate.test.ts @@ -0,0 +1,292 @@ +// `generate` for SQLite, checked on real SQLite: the migrations it writes, +// applied in order, must give the same PRAGMA results as creating the final +// schema fresh. +import { cpSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { Effect } from "effect" +import { Miniflare } from "miniflare" +import { afterEach, beforeEach, expect, it } from "vitest" +import { generate, readMigrations, type GenerateOptions, type KitConfig } from "../kit" +import * as Migrate from "../migrate" +import { STATEMENT_BREAKPOINT } from "../migrate/source" +import { renderSqliteSchema, sqliteEntitiesOf, type SqliteSchemaTable } from "../schema" +import { rowsAfter, shapeAfter } from "../sqlite/shape.test-support" + +const src = resolve(import.meta.dirname, "..") +let dir = "" +let version = 0 + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "effect-orm-sqlite-")) + version = 0 +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +/** Write a schema module (a fresh path each time, so the import cache never hands back an old one) and run generate on it. */ +const generateFrom = (body: string, options: GenerateOptions = {}) => { + const file = `schema${++version}.ts` + writeFileSync(join(dir, file), `import * as SQLite from "${src}/sqlite"\n${body}\n`) + const config: KitConfig = { dialect: "sqlite", schema: `./${file}`, out: "./migrations" } + return Effect.runPromise(generate(config, dir, { name: `v${version}`, ...options })) +} + +const folders = () => + readdirSync(join(dir, "migrations"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() +const migrationSql = (folder: string) => readFileSync(join(dir, "migrations", folder, "migration.sql"), "utf8") + +/** Every statement of every migration, in folder order, split as Alchemy and drizzle split them. */ +const allStatements = (seed: ReadonlyArray = []) => + folders().flatMap((folder, i) => [...(i === 1 ? seed : []), ...statementsOf(folder)]) + +const statementsOf = (folder: string) => ( + + migrationSql(folder) + .split(STATEMENT_BREAKPOINT) + .map((s) => s.trim()) + .filter((s) => s.replace(/^\s*--.*$/gm, "").trim().length > 0) +) + +/** The schema module's tables, created fresh. */ +const freshShape = async (body: string) => { + const file = `fresh${++version}.ts` + writeFileSync(join(dir, file), `import * as SQLite from "${src}/sqlite"\n${body}\n`) + const module = (await import(join(dir, file))) as Record> + return shapeAfter(await Effect.runPromise(Effect.map(sqliteEntitiesOf(Object.values(module)), renderSqliteSchema))) +} + +const expectMigratedToEqualFresh = async (body: string) => expect(await shapeAfter(allStatements())).toEqual(await freshShape(body)) + +const links = (columns: string, rest = "") => ` +export const Links = SQLite.table("links", { + columns: { ${columns} }, + primaryKey: ["id"], + ${rest} +})` + +it("writes a first migration as migration.sql plus a sqlite snapshot, then nothing for an unchanged schema", async () => { + const v1 = links(`id: SQLite.text, url: SQLite.text`) + const first = await generateFrom(v1) + expect(first.plan.map((p) => p.label)).toEqual(["metadata"]) + const [folder] = folders() + expect(folder).toMatch(/^\d{14}_v1$/) + expect(readdirSync(join(dir, "migrations", folder!)).sort()).toEqual(["migration.sql", "snapshot.json"]) + expect(JSON.parse(readFileSync(join(dir, "migrations", folder!, "snapshot.json"), "utf8")).dialect).toBe("sqlite") + expect(migrationSql(folder!)).toContain('CREATE TABLE IF NOT EXISTS "links"') + await expectMigratedToEqualFresh(v1) + expect((await generateFrom(v1)).written).toBeUndefined() +}) + +it("adds a nullable column, and a NOT NULL one with a literal default, with ALTER TABLE ADD COLUMN", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const v2 = links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text), hits: SQLite.column(SQLite.integer, { default: 0 })`) + const second = await generateFrom(v2) + expect(second.plan.flatMap((p) => p.sql)).toEqual([ + 'ALTER TABLE "links" ADD COLUMN "note" text', + 'ALTER TABLE "links" ADD COLUMN "hits" integer NOT NULL DEFAULT 0', + ]) + await expectMigratedToEqualFresh(v2) +}) + +const seedLinks = [`INSERT INTO "links" ("id", "url") VALUES ('a', 'https://a'), ('b', 'https://b')`] + +it("drops a column only once the data loss is confirmed, and keeps the other columns' rows", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text)`)) + const v2 = links(`id: SQLite.text, url: SQLite.text`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + expect(folders()).toHaveLength(1) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.note" }] }) + expect(second.plan).toEqual([{ label: "destructive", sql: ['ALTER TABLE "links" DROP COLUMN "note"'] }]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT * FROM "links" ORDER BY "id"')).toEqual([ + { id: "a", url: "https://a" }, + { id: "b", url: "https://b" }, + ]) +}) + +const rebuilds: ReadonlyArray = [ + ["a column type", `id: SQLite.text, url: SQLite.text, hits: SQLite.nullable(SQLite.text)`, `id: SQLite.text, url: SQLite.text, hits: SQLite.nullable(SQLite.integer)`], + ["nullability", `id: SQLite.text, url: SQLite.nullable(SQLite.text)`, `id: SQLite.text, url: SQLite.text`], + ["a default", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.column(SQLite.text, { default: "none" })`], + ["a UNIQUE column", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.column(SQLite.text, { unique: true })`], + ["a NOT NULL column with an expression default", `id: SQLite.text, url: SQLite.text`, `id: SQLite.text, url: SQLite.text, at: SQLite.column(SQLite.text, { defaultExpr: "(CURRENT_TIMESTAMP)" })`], +] + +it.each(rebuilds)("rebuilds the table for %s, keeps its rows and recreates its indexes", async (_, before, after) => { + const index = `indexes: [SQLite.index("links_url", ["url"])],` + await generateFrom(links(before, index)) + const v2 = links(after, index) + const second = await generateFrom(v2) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild", "metadata"]) + expect(second.plan[0]!.sql).toEqual([ + "PRAGMA foreign_keys=OFF", + expect.stringMatching(/^CREATE TABLE "__new_links" \(/), + expect.stringMatching(/^INSERT INTO "__new_links"\("id", "url"(, "hits")?\) SELECT "id", "url"(, "hits")? FROM "links"$/), + 'DROP TABLE "links"', + 'ALTER TABLE "__new_links" RENAME TO "links"', + "PRAGMA foreign_keys=ON", + ]) + expect(second.plan[1]!.sql).toEqual(['CREATE INDEX IF NOT EXISTS "links_url" ON "links" ("url")']) + await expectMigratedToEqualFresh(v2) + const rows = await rowsAfter(allStatements(seedLinks), 'SELECT "id", "url" FROM "links" ORDER BY "id"') + expect(rows).toEqual([{ id: "a", url: "https://a" }, { id: "b", url: "https://b" }]) +}) + +it("rebuilds the table for a primary key change", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const v2 = ` +export const Links = SQLite.table("links", { + columns: { id: SQLite.text, url: SQLite.text }, + primaryKey: ["id", "url"], +})` + const second = await generateFrom(v2) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild"]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT count(*) AS n FROM "links"')).toEqual([{ n: 2 }]) +}) + +it("creates and drops indexes without touching the table", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, org: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + const v2 = links(`id: SQLite.text, url: SQLite.text, org: SQLite.text`, `indexes: [SQLite.uniqueIndex("links_org_url", ["org", "url"])],`) + const second = await generateFrom(v2) + expect(second.plan).toEqual([ + { label: "metadata", sql: ['DROP INDEX IF EXISTS "links_url"'] }, + { label: "metadata", sql: ['CREATE UNIQUE INDEX IF NOT EXISTS "links_org_url" ON "links" ("org", "url")'] }, + ]) + await expectMigratedToEqualFresh(v2) +}) + +it("asks to confirm a column a rebuild leaves behind", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text, note: SQLite.nullable(SQLite.text)`)) + const v2 = links(`id: SQLite.text, url: SQLite.integer`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints", details: [expect.stringContaining("links.note")] }) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "column", entity: "links.note" }] }) + expect(second.plan.map((p) => p.label)).toEqual(["rebuild"]) + expect(second.plan[0]!.sql[2]).toBe('INSERT INTO "__new_links"("id", "url") SELECT "id", "url" FROM "links"') + await expectMigratedToEqualFresh(v2) +}) + +it("drops a table only once the data loss is confirmed", async () => { + await generateFrom(`${links(`id: SQLite.text, url: SQLite.text`)}\nexport const Old = SQLite.table("old", { columns: { id: SQLite.text }, indexes: [SQLite.index("old_id", ["id"])] })`) + const v2 = links(`id: SQLite.text, url: SQLite.text`) + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + const second = await generateFrom(v2, { hints: [{ type: "confirm_data_loss", kind: "table", entity: "old" }] }) + expect(second.plan).toEqual([{ label: "destructive", sql: ['DROP TABLE IF EXISTS "old"'] }]) + await expectMigratedToEqualFresh(v2) +}) + +it("renames a table and a column when a hint says so, keeping rows and indexes", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + const v2 = ` +export const Urls = SQLite.table("urls", { + columns: { id: SQLite.text, target: SQLite.text }, + primaryKey: ["id"], + indexes: [SQLite.index("links_url", ["target"])], +})` + // Unconfirmed, a rename is a drop plus a create. + await expect(generateFrom(v2)).rejects.toMatchObject({ code: "missing_hints" }) + const second = await generateFrom(v2, { + hints: [ + { type: "confirm_rename", kind: "table", entity: "links", to: "urls" }, + { type: "confirm_rename", kind: "column", entity: "urls.url", to: "target" }, + ], + }) + expect(second.plan).toEqual([ + { label: "metadata", sql: ['ALTER TABLE "links" RENAME TO "urls"'] }, + { label: "metadata", sql: ['ALTER TABLE "urls" RENAME COLUMN "url" TO "target"'] }, + ]) + await expectMigratedToEqualFresh(v2) + expect(await rowsAfter(allStatements(seedLinks), 'SELECT "target" FROM "urls" ORDER BY "id"')).toEqual([{ target: "https://a" }, { target: "https://b" }]) +}) + +// The tables of __fixtures__/drizzle-sqlite/schema.ts.txt, declared with effect-orm. +const drizzleEquivalent = ` +const named = [0]>(type: T, name: string) => SQLite.column(type, { name }) +const optionalText = (name: string) => named(SQLite.nullable(SQLite.text), name) +export const ShortLink = SQLite.table("short_link", { + columns: { + id: SQLite.text, code: SQLite.text, targetUrl: named(SQLite.text, "target_url"), targetHash: named(SQLite.text, "target_hash"), + orgId: optionalText("org_id"), createdBy: optionalText("created_by"), subjectType: named(SQLite.text, "subject_type"), + subjectId: named(SQLite.text, "subject_id"), purpose: SQLite.text, prospectId: optionalText("prospect_id"), + clickCount: named(SQLite.integer, "click_count"), firstAccessedAt: optionalText("first_accessed_at"), + lastAccessedAt: optionalText("last_accessed_at"), createdAt: named(SQLite.text, "created_at"), updatedAt: named(SQLite.text, "updated_at"), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], +}) +export const Membership = SQLite.table("membership", { + columns: { + orgId: named(SQLite.text, "org_id"), userId: named(SQLite.text, "user_id"), + role: SQLite.column(SQLite.text, { default: "member" }), weight: SQLite.column(SQLite.real, { default: 1.5 }), + active: SQLite.column(SQLite.boolean, { default: true }), avatar: SQLite.nullable(SQLite.blob), + amount: SQLite.nullable(SQLite.custom("numeric", SQLite.real.schema)), + email: SQLite.column(SQLite.nullable(SQLite.text), { unique: true }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["orgId", "userId"], + indexes: [SQLite.index("membership_user_role", ["userId", "role"])], +})` + +it("adopts a drizzle-kit 1.0 folder: a baseline from its snapshot, then no change for the same tables", async () => { + cpSync(join(import.meta.dirname, "__fixtures__", "drizzle-sqlite"), join(dir, "migrations"), { recursive: true }) + await expect(generateFrom(drizzleEquivalent)).rejects.toMatchObject({ code: "config", message: expect.stringContaining("--baseline") }) + const baseline = await generateFrom(drizzleEquivalent, { baseline: "drizzle" }) + expect(baseline.plan).toEqual([]) + const [, adopted] = folders() + expect(migrationSql(adopted!)).toMatch(/^-- Baseline/) + expect(JSON.parse(readFileSync(join(dir, "migrations", adopted!, "snapshot.json"), "utf8")).dialect).toBe("sqlite") + expect(await generateFrom(drizzleEquivalent)).toEqual({ written: undefined, plan: [] }) + // The comparison is live: one difference from the drizzle tables is a change. + const changed = await generateFrom(drizzleEquivalent.replace('"short_link_org", ["orgId"]', '"short_link_org", ["orgId", "purpose"]')) + expect(changed.plan.flatMap((p) => p.sql)).toEqual([ + 'DROP INDEX IF EXISTS "short_link_org"', + 'CREATE INDEX IF NOT EXISTS "short_link_org" ON "short_link" ("org_id", "purpose")', + ]) + // The folder drizzle-kit wrote, applied as it is, holds the same tables apart from the NOT NULL + // drizzle-kit drops from a text primary key. + expect(Object.keys(await shapeAfter(allStatements()))).toEqual(["membership", "short_link"]) +}) + +it("writes migrations a local D1 database accepts, rebuilds included", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`, `indexes: [SQLite.index("links_url", ["url"])],`)) + await generateFrom(links(`id: SQLite.text, url: SQLite.column(SQLite.text, { unique: true }), hits: SQLite.column(SQLite.integer, { default: 0 })`, `indexes: [SQLite.index("links_url", ["url"])],`)) + expect(folders()).toHaveLength(2) + const mf = new Miniflare({ modules: true, script: "export default { fetch() { return new Response('') } }", compatibilityDate: "2026-07-01", d1Databases: ["DB"] }) + try { + const db = await mf.getD1Database("DB") + const [first, second] = folders() + await db.batch(statementsOf(first!).map((sql) => db.prepare(sql))) + await db.batch(seedLinks.map((sql) => db.prepare(sql))) + // One atomic batch per migration, as a D1 migration runner sends it. + await db.batch(statementsOf(second!).map((sql) => db.prepare(sql))) + const rows = await db.prepare('SELECT "id", "url", "hits" FROM "links" ORDER BY "id"').all() + expect(rows.results).toEqual([{ id: "a", url: "https://a", hits: 0 }, { id: "b", url: "https://b", hits: 0 }]) + const indexes = await db.prepare(`SELECT name, origin FROM pragma_index_list('links') ORDER BY name`).all() + expect(indexes.results).toEqual([ + { name: "links_url", origin: "c" }, + { name: "sqlite_autoindex_links_1", origin: "pk" }, + { name: "sqlite_autoindex_links_2", origin: "u" }, + ]) + } finally { + await mf.dispose() + } +}) + +it("refuses to run, record or verify SQLite migrations with the effect-orm runner, which has no SQLite ledger yet", async () => { + await generateFrom(links(`id: SQLite.text, url: SQLite.text`)) + const migrations = await Effect.runPromise(readMigrations(join(dir, "migrations"))) + const driver = Effect.provideService(Migrate.MigrationDriver, {} as Migrate.MigrationDriver["Service"]) + for (const attempt of [ + Migrate.run({ migrations }), + Migrate.status(migrations), + Migrate.verify(migrations), + Migrate.baseline({ migrations, upTo: migrations[0]!.name }), + ]) { + await expect(Effect.runPromise(driver(attempt as Effect.Effect))).rejects.toMatchObject({ + message: expect.stringContaining("SQLite migrations are not run by effect-orm yet"), + }) + } +}) diff --git a/src/migrate/run.ts b/src/migrate/run.ts index 3b7efdf..e39bdbd 100644 --- a/src/migrate/run.ts +++ b/src/migrate/run.ts @@ -27,7 +27,7 @@ import { } from "./ledger" import { ensurePgLedger, isPgApplied, lockPg, readPgApplied, recordPgMigration, transactionOf } from "./pg-ledger" import { backfillWindows, renderBackfill, renderBackfillBounds, renderOp } from "../schema/ops" -import { dialectOf, stepsOf, type LoadedMigration, type MigrationStep } from "./source" +import { dialectOf, refuseSqlite, stepsOf, type LoadedMigration, type MigrationStep } from "./source" export interface RunOptions { readonly migrations: ReadonlyArray @@ -49,6 +49,7 @@ export interface RunOptions { const resolveDialect = (migrations: ReadonlyArray, dialect: SchemaDialect | undefined): SchemaDialect => dialect ?? dialectOf(migrations) ?? "clickhouse" + export interface AppliedMigration { readonly name: string readonly steps: number @@ -250,7 +251,9 @@ const applyOne = (migration: LoadedMigration, render: RenderOptions, renew: Effe * have their hash checked. Returns what ran. */ export const run = (options: RunOptions): Effect.Effect, MigrateError, MigrationDriver> => - resolveDialect(options.migrations, options.dialect) === "postgres" ? runPg(options) : runClickHouse(options) + Effect.andThen(refuseSqlite(options.migrations, options.dialect), () => + resolveDialect(options.migrations, options.dialect) === "postgres" ? runPg(options) : runClickHouse(options), + ) const runClickHouse = (options: RunOptions): Effect.Effect, MigrateError, MigrationDriver> => Effect.gen(function* () { @@ -335,6 +338,7 @@ export const baseline = (options: BaselineOptions): Effect.Effect m.name === options.upTo) if (index === -1) return yield* new MigrateSourceError({ migration: options.upTo, message: "is not one of the migrations" }) + yield* refuseSqlite(options.migrations, options.dialect) const postgres = resolveDialect(options.migrations, options.dialect) === "postgres" if (postgres) yield* ensurePgLedger else yield* ensureLedger(options.render ?? {}) @@ -355,6 +359,7 @@ export const status = ( dialect?: SchemaDialect, ): Effect.Effect, MigrateError, MigrationDriver> => Effect.gen(function* () { + yield* refuseSqlite(migrations, dialect) if (resolveDialect(migrations, dialect) === "postgres") { yield* ensurePgLedger const applied = new Map((yield* readPgApplied).map((row) => [row.name, row])) @@ -402,6 +407,7 @@ export interface ResolveStepOptions { export const resolveStep = (options: ResolveStepOptions): Effect.Effect => Effect.gen(function* () { const { migration } = options + yield* refuseSqlite([migration], options.dialect) if (resolveDialect([migration], options.dialect) === "postgres") { return yield* new MigrateSourceError({ migration: migration.name, diff --git a/src/migrate/source.ts b/src/migrate/source.ts index 11e1e16..63897ae 100644 --- a/src/migrate/source.ts +++ b/src/migrate/source.ts @@ -86,6 +86,10 @@ const foreignSnapshotOf = (text: string): LoadedMigration["foreignSnapshot"] => : undefined } +/** The dialect a drizzle-kit snapshot names: `sqlite` (or `turso`) is SQLite; anything else is read as Postgres, as before. */ +const drizzleDialect = (json: unknown): SchemaDialect => + typeof json === "object" && json !== null && "dialect" in json && (json.dialect === "sqlite" || json.dialect === "turso") ? "sqlite" : "postgres" + /** * Each migration's parent migrations, from its snapshot's `prevIds`. A custom * migration copies its parent's entities, so several migrations can share one @@ -212,7 +216,23 @@ export const dialectOf = (migrations: ReadonlyArray): SchemaDia for (const m of migrations) { if (m.snapshot !== undefined) return m.snapshot.dialect if (m.file !== undefined) return "dialect" in m.file ? "postgres" : "clickhouse" - if (m.foreignSnapshot !== undefined) return "postgres" + if (m.foreignSnapshot !== undefined) return drizzleDialect(m.foreignSnapshot.json) } return undefined } + +/** + * `generate` writes SQLite migrations, but this runner does not apply them yet: + * apply the folder with a tool that reads drizzle-kit folders (Alchemy's D1 + * resource, drizzle's migrator). Refused here rather than run with the + * ClickHouse ledger. + */ +export const refuseSqlite = (migrations: ReadonlyArray, dialect: SchemaDialect | undefined) => + (dialect ?? dialectOf(migrations)) === "sqlite" + ? Effect.fail( + new MigrateSourceError({ + migration: migrations[0]?.name ?? "(none)", + message: "SQLite migrations are not run by effect-orm yet; apply the folder with a tool that reads drizzle-kit folders", + }), + ) + : Effect.void diff --git a/src/migrate/verify.ts b/src/migrate/verify.ts index b5b02d2..1ee74d0 100644 --- a/src/migrate/verify.ts +++ b/src/migrate/verify.ts @@ -19,7 +19,7 @@ import type { MigrateSourceError, MigrateSqlError } from "./errors" import { LEDGER_TABLES, readApplied } from "./ledger" import { readPgApplied } from "./pg-ledger" import { verifyPg, type PgVerifyOptions } from "./pg-verify" -import { dialectOf, type LoadedMigration } from "./source" +import { dialectOf, refuseSqlite, type LoadedMigration } from "./source" export interface Drift { readonly entity: string @@ -116,6 +116,7 @@ export const verify = ( options: VerifyOptions = {}, ): Effect.Effect => Effect.gen(function* () { + yield* refuseSqlite(migrations, options.dialect) const postgres = (options.dialect ?? dialectOf(migrations)) === "postgres" const appliedRows = yield* (postgres ? readPgApplied : readApplied).pipe(Effect.orElseSucceed(() => [])) const appliedNames = new Set(appliedRows.map((row) => row.name)) @@ -123,6 +124,7 @@ export const verify = ( const snapshot = against?.snapshot if (against === undefined || snapshot === undefined) return { against: undefined, drift: [] } if (snapshot.dialect === "postgres") return { against: against.name, drift: yield* verifyPg(snapshot, against.name, options) } + if (snapshot.dialect === "sqlite") return yield* refuseSqlite(migrations, "sqlite").pipe(Effect.as({ against: undefined, drift: [] })) return { against: against.name, drift: yield* verifyClickHouse(snapshot) } }) diff --git a/src/schema.ts b/src/schema.ts index 7fe9f4d..0afb383 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -23,6 +23,7 @@ export { SchemaEntity, Snapshot, SNAPSHOT_VERSION, + SqliteSnapshot, TableEntity, canonicalJson, entityKey, @@ -87,5 +88,31 @@ export { type PgOpLabel, } from "./schema/pg-ops" export { diffPgSchemas } from "./schema/pg-diff" +export { sqliteEntitiesOf, type SqliteSchemaTable } from "./schema/sqlite-define" +export { + SqliteColumnEntity, + SqliteIndexEntity, + SqliteSchemaEntity, + SqliteTableEntity, + SqliteUniqueEntity, + canonicalSqliteType, +} from "./schema/sqlite-entities" +export { diffSqliteSchemas } from "./schema/sqlite-diff" +export { fromDrizzleSqliteSnapshot, type DrizzleSqliteImport } from "./schema/sqlite-drizzle" +export { + SqliteMigrationOp, + labelOfSqlite, + rebuildName, + renderSqliteMigrationSql, + renderSqliteOp, + type SqliteOpLabel, +} from "./schema/sqlite-ops" +export { + renderSqliteColumnDefinition, + renderSqliteCreateIndex, + renderSqliteCreateTable, + renderSqliteSchema, + sqliteIdent, +} from "./schema/sqlite-render" export { fromDrizzleSnapshot, type DrizzleImport } from "./schema/drizzle" export { Hint, Hints, diffSchemas, type DiffResult, type UnsupportedChange } from "./schema/diff" diff --git a/src/schema/ddl-expr.ts b/src/schema/ddl-expr.ts new file mode 100644 index 0000000..a8afc6e --- /dev/null +++ b/src/schema/ddl-expr.ts @@ -0,0 +1,34 @@ +// Expressions in DDL: defaults, index keys and predicates, written with the +// query DSL or as SQL strings, and rendered once with the dialect the table is +// for. Shared by the Postgres and SQLite `table`. + +import { compile as compileFragment } from "../sql/sql-fragment" +import { withDialect, type Dialect } from "../ch/dialect" +import type { Condition, Expr } from "../ch/expr" +import { createColumnAccessor, type ColumnAccessor } from "../ch/query" +import { columnSqlName, type ColumnDefs } from "../ch/types" +import type { DdlExpr, DdlKey } from "./define" + +/** A partial index's predicate: SQL, or a condition built with the DSL. */ +export type DdlPredicate = string | (($: ColumnAccessor) => Condition | Expr | string) + +export interface DdlRenderer { + /** SQL as written, or a DSL value rendered with the dialect. */ + readonly value: (value: Expr | Condition | string) => string + readonly expr: (expr: DdlExpr | DdlPredicate, columns: Cols) => string + /** Key parts: a column name is written quoted, an expression as it renders. */ + readonly keyParts: (key: DdlKey, columns: Cols) => ReadonlyArray +} + +export const ddlRenderer = (dialect: Dialect): DdlRenderer => { + const value = (v: Expr | Condition | string): string => + typeof v === "string" ? v : withDialect(dialect, () => compileFragment(v.toFragment())) + return { + value, + expr: (expr, columns) => (typeof expr === "string" ? expr : value(expr(createColumnAccessor(columns)))), + keyParts: (key, columns) => + typeof key === "function" + ? key(createColumnAccessor(columns)).map(value) + : key.map((column) => dialect.quoteIdent(columnSqlName(columns, column))), + } +} diff --git a/src/schema/diff.ts b/src/schema/diff.ts index 2abc6bd..a83b88e 100644 --- a/src/schema/diff.ts +++ b/src/schema/diff.ts @@ -33,11 +33,15 @@ export const Hint = Schema.Union([ /** `table`, `table.column`, or the enum's name. */ entity: Schema.String, }), - /** A Postgres enum label renamed in place (`ALTER TYPE ... RENAME VALUE`): rows holding it now hold `to`. */ + /** + * A rename. `enum_value`: a Postgres enum label renamed in place (`ALTER + * TYPE ... RENAME VALUE`), so rows holding it now hold `to`. `table` and + * `column` (SQLite): `ALTER TABLE ... RENAME`, instead of a drop and a create. + */ Schema.Struct({ type: Schema.Literal("confirm_rename"), - kind: Schema.Literal("enum_value"), - /** `.`. */ + kind: Schema.Literals(["enum_value", "table", "column"]), + /** `.`, the old table name, or `
.` with the table's new name. */ entity: Schema.String, to: Schema.String, }), diff --git a/src/schema/entities.ts b/src/schema/entities.ts index 66788cd..5b4dc04 100644 --- a/src/schema/entities.ts +++ b/src/schema/entities.ts @@ -9,6 +9,7 @@ import { Schema } from "effect" import { PgSchemaEntity } from "./pg-entities" +import { SqliteSchemaEntity } from "./sqlite-entities" /** `DEFAULT`, `MATERIALIZED`, or `ALIAS`, with its SQL expression. */ export const ColumnDefault = Schema.Struct({ @@ -102,15 +103,22 @@ export const PgSnapshot = Schema.Struct({ }) export type PgSnapshot = typeof PgSnapshot.Type +export const SqliteSnapshot = Schema.Struct({ + ...snapshotFields, + dialect: Schema.Literal("sqlite"), + entities: Schema.Array(SqliteSchemaEntity), +}) +export type SqliteSnapshot = typeof SqliteSnapshot.Type + /** The schema at one point in history. `dialect` says which entity set it holds. */ -export const Snapshot = Schema.Union([ClickHouseSnapshot, PgSnapshot]) +export const Snapshot = Schema.Union([ClickHouseSnapshot, PgSnapshot, SqliteSnapshot]) export type Snapshot = typeof Snapshot.Type /** The dialects a schema can be written for. */ export type SchemaDialect = Snapshot["dialect"] -/** An entity of either dialect. Everything dialect-neutral (keys, ordering, hashing, the branch graph) takes this. */ -export type AnySchemaEntity = SchemaEntity | PgSchemaEntity +/** An entity of any dialect. Everything dialect-neutral (keys, ordering, hashing, the branch graph) takes this. */ +export type AnySchemaEntity = SchemaEntity | PgSchemaEntity | SqliteSchemaEntity /** A stable key per entity, unique within a snapshot: `:`, or `:
.` for a table's parts. */ export const entityKey = (entity: AnySchemaEntity): string => diff --git a/src/schema/pg-define.ts b/src/schema/pg-define.ts index 956d461..4d14b5e 100644 --- a/src/schema/pg-define.ts +++ b/src/schema/pg-define.ts @@ -8,15 +8,13 @@ // become DDL records a problem, and `pgEntitiesOf` fails with it. import { Schema } from "effect" -import { compile as compileFragment } from "../sql/sql-fragment" import { withDialect } from "../ch/dialect" -import type { Condition, Expr } from "../ch/expr" import { encodeColumnLiteral } from "../ch/literal" -import { createColumnAccessor, type ColumnAccessor } from "../ch/query" import type { Table } from "../ch/table" import { columnSqlName, type CHType, type ColumnDefs, type InferTS } from "../ch/types" import type { PgType } from "../pg/types" import { postgresDialect } from "../pg/dialect" +import { ddlRenderer, type DdlPredicate } from "./ddl-expr" import { externalTable, type DdlExpr, type DdlKey } from "./define" import { withRenderProblems, type DefinitionProblem, type ProblemSink } from "./problems" import { @@ -33,22 +31,11 @@ import { type PgUniqueEntity, } from "./pg-entities" -const quoteIdent = (name: string): string => `"${name.replace(/"/g, '""')}"` +const ddl = ddlRenderer(postgresDialect) +const renderExpr = ddl.expr +const renderKeyParts = ddl.keyParts -const renderValue = (value: Expr | Condition | string): string => - typeof value === "string" ? value : withDialect(postgresDialect, () => compileFragment(value.toFragment())) - -/** A partial index's predicate: SQL, or a condition built with the DSL. */ -export type DdlPredicate = string | (($: ColumnAccessor) => Condition | Expr | string) - -const renderExpr = (expr: DdlExpr | DdlPredicate, columns: Cols): string => - typeof expr === "string" ? expr : renderValue(expr(createColumnAccessor(columns))) - -/** Key parts: a column name is written quoted, an expression as it renders. */ -const renderKeyParts = (key: DdlKey, columns: Cols): ReadonlyArray => - typeof key === "function" - ? key(createColumnAccessor(columns)).map(renderValue) - : key.map((column) => quoteIdent(columnSqlName(columns, column))) +export type { DdlPredicate } // Columns diff --git a/src/schema/snapshot.ts b/src/schema/snapshot.ts index 181d249..5495775 100644 --- a/src/schema/snapshot.ts +++ b/src/schema/snapshot.ts @@ -15,12 +15,20 @@ import { type SchemaDialect, type SchemaEntity, type Snapshot, + type SqliteSnapshot, } from "./entities" import { enumProblems, type PgEnum, type PgSchemaTable } from "./pg-define" import type { PgEnumEntity, PgSchemaEntity } from "./pg-entities" +import type { SqliteSchemaTable } from "./sqlite-define" +import type { SqliteSchemaEntity } from "./sqlite-entities" /** Anything a schema module may export that `generate` collects. */ -export type SchemaObject = SchemaTable | MaterializedView | PgSchemaTable | PgEnum +export type SchemaObject = + | SchemaTable + | MaterializedView + | PgSchemaTable + | PgEnum + | SqliteSchemaTable export const isSchemaObject = (value: unknown): value is SchemaObject => typeof value === "object" && @@ -47,7 +55,7 @@ export const entitiesOf = (objects: ReadonlyArray): Effect.Effect< for (const object of objects) { problems.push(...object.problems) if (dialectOfObject(object) !== "clickhouse") { - problems.push({ object: object.name, message: "is a Postgres table; use pgEntitiesOf" }) + problems.push({ object: object.name, message: `is a ${dialectOfObject(object) === "postgres" ? "Postgres" : "SQLite"} table; use ${dialectOfObject(object) === "postgres" ? "pgEntitiesOf" : "sqliteEntitiesOf"}` }) continue } if (object._tag === "Table") { @@ -101,7 +109,7 @@ export const pgEntitiesOf = (objects: ReadonlyArray): Effect.Effec for (const object of objects) { if (dialectOfObject(object) !== "postgres") { problems.push(...object.problems) - problems.push({ object: object.name, message: "is a ClickHouse definition; use entitiesOf" }) + problems.push({ object: object.name, message: dialectOfObject(object) === "sqlite" ? "is a SQLite table; use sqliteEntitiesOf" : "is a ClickHouse definition; use entitiesOf" }) continue } if (object._tag === "PgEnum") { @@ -175,6 +183,11 @@ export function makeSnapshot( prevIds: ReadonlyArray, dialect: "postgres", ): Effect.Effect +export function makeSnapshot( + entities: ReadonlyArray, + prevIds: ReadonlyArray, + dialect: "sqlite", +): Effect.Effect export function makeSnapshot( entities: ReadonlyArray, prevIds: ReadonlyArray, diff --git a/src/schema/sqlite-define.ts b/src/schema/sqlite-define.ts new file mode 100644 index 0000000..ca30f39 --- /dev/null +++ b/src/schema/sqlite-define.ts @@ -0,0 +1,323 @@ +// SQLite schema definitions: `table` from `/sqlite` with its column and index +// helpers. +// +// The SQLite counterpart of the Postgres `table`: the value IS a `Table`, so +// every query API accepts it, and its DDL rides beside it on `ddl` as SQLite +// entities. A definition that cannot become DDL records a problem, and +// `sqliteEntitiesOf` fails with it. + +import { Effect } from "effect" +import type { Table } from "../ch/table" +import { withDialect } from "../ch/dialect" +import { encodeColumnLiteral } from "../ch/literal" +import { columnSqlName, type CHType, type ColumnDefs, type InferTS } from "../ch/types" +import { sqliteDialect } from "../sqlite/dialect" +import { ddlRenderer } from "./ddl-expr" +import { externalTable, type DdlExpr, type DdlKey } from "./define" +import { sortEntities } from "./entities" +import { checkIdentifier, definitionError, withRenderProblems, type DefinitionProblem, type ProblemSink, type SchemaDefinitionError } from "./problems" +import { + canonicalSqliteType, + type SqliteColumnEntity, + type SqliteIndexEntity, + type SqliteSchemaEntity, + type SqliteTableEntity, + type SqliteUniqueEntity, +} from "./sqlite-entities" + +const ddl = ddlRenderer(sqliteDialect) + +// Columns + +export interface ColumnOptions> { + /** + * The column's name in the database, when it differs from its key in + * `columns`: `orgId: SQLite.column(SQLite.text, { name: "org_id" })` is read + * and written as `$.orgId` and stored as `org_id`. + */ + readonly name?: string + /** A literal default, encoded through the column's own type. A boolean is written `true` / `false`. */ + readonly default?: InferTS + /** + * `DEFAULT `. A string is written as given, so put an expression in + * the parentheses SQLite requires: `"(unixepoch())"`. A DSL expression is + * parenthesized for you. + */ + readonly defaultExpr?: DdlExpr + /** + * A one-column `UNIQUE` constraint, written inline as drizzle-kit 1.0 writes + * `.unique()`. `true` names it `
__unique`, drizzle's name; a + * string names it. SQLite keeps no constraint name; the snapshot does. + */ + readonly unique?: true | string +} + +/** `Given` records which options were set, so the table knows which columns an insert may leave out. */ +export interface ColumnSpec< + T extends CHType, + Given extends keyof ColumnOptions = keyof ColumnOptions, +> { + readonly _tag: "SqliteColumnSpec" + readonly type: T + readonly options: ColumnOptions + readonly _given?: Given +} + +/** A column with options. A bare column type works too where none is needed. */ +export const column = , Given extends keyof ColumnOptions = never>( + type: T, + options: ColumnOptions & { readonly [K in Given]: unknown } = {} as ColumnOptions & { + readonly [K in Given]: unknown + }, +): ColumnSpec => ({ _tag: "SqliteColumnSpec", type, options }) + +export type ColumnInput = CHType | ColumnSpec, any> + +/** The query-side column types of a `columns` record. */ +export type ColumnsOf> = { + readonly [K in keyof I]: I[K] extends ColumnSpec ? T : I[K] extends CHType ? I[K] : never +} + +/** Columns declared with `default` or `defaultExpr`: an insert may leave them out. */ +export type DefaultedColumnsOf> = { + [K in keyof I]: I[K] extends ColumnSpec + ? [Exclude] extends [never] + ? never + : K + : never +}[keyof I] & + string + +const isColumnSpec = (input: unknown): input is ColumnSpec, any> => + typeof input === "object" && input !== null && "_tag" in input && input._tag === "SqliteColumnSpec" + +const columnTypeOfInput = (input: ColumnInput): CHType => { + if (!isColumnSpec(input)) return input + return input.options.name === undefined ? input.type : { ...input.type, sqlName: input.options.name } +} + +// Indexes + +export interface IndexSpec { + readonly _tag: "SqliteIndexSpec" + readonly name: string + readonly unique: boolean + readonly on: DdlKey +} + +/** An index on columns or expressions: `index("t_org", ["orgId"])`, or a callback building expressions. */ +export const index = (name: string, on: DdlKey): IndexSpec => ({ + _tag: "SqliteIndexSpec", + name, + unique: false, + on, +}) + +/** A unique index. An `onConflict` target can name its columns. */ +export const uniqueIndex = (name: string, on: DdlKey): IndexSpec => ({ + _tag: "SqliteIndexSpec", + name, + unique: true, + on, +}) + +const columnDefault = ( + problems: ProblemSink, + table: string, + name: string, + spec: ColumnSpec>, + columns: ColumnDefs, +): string | null => { + const { options } = spec + if (options.default !== undefined && options.defaultExpr !== undefined) { + problems.push({ object: `${table}.${name}`, message: "a column takes one of default, defaultExpr" }) + } + if (options.defaultExpr !== undefined) { + return typeof options.defaultExpr === "string" ? options.defaultExpr : `(${ddl.expr(options.defaultExpr, columns)})` + } + if (options.default === undefined || options.default === null) return null + // SQLite reads TRUE / FALSE as 1 / 0; drizzle-kit writes them as keywords. + if (typeof options.default === "boolean") return options.default ? "true" : "false" + return withDialect(sqliteDialect, () => encodeColumnLiteral(spec.type, options.default, name)) +} + +// Tables + +export interface TableDefinition> { + readonly columns: Columns + /** One column, written inline as drizzle-kit does, or several, as a table constraint. */ + readonly primaryKey?: ReadonlyArray + readonly indexes?: ReadonlyArray>> + /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ + readonly tenantColumn?: keyof Columns & string +} + +/** A table this schema does not own: a view, or one another tool migrates. No DDL. */ +export interface ExternalTableDefinition> { + readonly external: true + readonly columns: Columns + readonly tenantColumn?: keyof Columns & string +} + +/** The DDL a SQLite table carries, as entities. */ +export interface TableDdl { + readonly dialect: "sqlite" + readonly table: SqliteTableEntity + readonly columns: ReadonlyArray + readonly indexes: ReadonlyArray + readonly uniques: ReadonlyArray +} + +export interface SqliteSchemaTable + extends Table { + readonly ddl: TableDdl + /** What is wrong with the definition; `sqliteEntitiesOf` fails when any table has one. */ + readonly problems: ReadonlyArray +} + +/** A table or index name SQLite reserves for itself. */ +const checkName = (problems: ProblemSink, object: string, name: string): void => { + checkIdentifier(problems, object, name) + if (name.toLowerCase().startsWith("sqlite_")) { + problems.push({ object, message: `${JSON.stringify(name)} starts with sqlite_, which SQLite reserves` }) + } +} + +/** Is `definition` a `{ columns, ... }` definition, rather than a record of columns? */ +export const isTableDefinition = (definition: object): definition is { readonly columns: Record } => { + if (!("columns" in definition)) return false + const columns = definition.columns + return typeof columns === "object" && columns !== null && !("sql" in columns && "schema" in columns) +} + +export const externalSqliteTable = ( + name: Name, + definition: ExternalTableDefinition>, +): Table => externalTable(name, definition, isColumnSpec, [], (spec) => columnTypeOfInput(spec as ColumnInput)) + +/** + * A SQLite table with its DDL. The value IS a query `Table`. + * + * A column is `NOT NULL` unless its type is `SQLite.nullable(...)`. + */ +export const sqliteTable = >( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> => withRenderProblems(name, () => buildTable(name, definition)) + +function buildTable>( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> { + const problems: ProblemSink = [] + checkName(problems, name, name) + const inputs = Object.entries(definition.columns) + if (inputs.length === 0) problems.push({ object: name, message: "a table needs columns" }) + const types = Object.fromEntries(inputs.map(([column, input]) => [column, columnTypeOfInput(input)])) as ColumnsOf + const sqlName = (column: string) => columnSqlName(types, column) + const has = (column: string) => Object.hasOwn(types, column) + + const columnEntities = inputs.map(([column, input], position): SqliteColumnEntity => { + checkIdentifier(problems, `${name}.${column}`, sqlName(column)) + const spec: ColumnSpec> = isColumnSpec(input) + ? input + : { _tag: "SqliteColumnSpec", type: input as CHType, options: {} } + return { + kind: "column", + table: name, + name: sqlName(column), + position, + type: canonicalSqliteType(spec.type.sql), + notNull: spec.type._tag !== "Nullable", + default: columnDefault(problems, name, column, spec, types), + } + }) + + const pkColumns = definition.primaryKey ?? [] + if (definition.primaryKey !== undefined) { + if (pkColumns.length === 0) problems.push({ object: name, message: "a primary key needs columns" }) + for (const column of pkColumns) { + if (!has(column)) problems.push({ object: `${name} primary key`, message: `${column} is not a column` }) + else if (columnEntities.find((c) => c.name === sqlName(column))?.notNull === false) { + problems.push({ + object: `${name}.${column}`, + message: "a primary key column cannot be nullable; SQLite would accept NULL keys", + }) + } + } + } + + const indexEntities: ReadonlyArray = (definition.indexes ?? []).map((spec): SqliteIndexEntity => { + checkName(problems, `${name} index`, spec.name) + if (Array.isArray(spec.on)) { + for (const column of spec.on) { + if (!has(column)) problems.push({ object: `${name} index ${spec.name}`, message: `${column} is not a column` }) + } + } + const columns = ddl.keyParts(spec.on, types) + if (columns.length === 0) problems.push({ object: `${name} index ${spec.name}`, message: "an index needs columns" }) + return { kind: "index", table: name, name: spec.name, unique: spec.unique, columns } + }) + + const uniqueEntities: Array = [] + for (const [column, input] of inputs) { + const option = isColumnSpec(input) ? input.options.unique : undefined + if (option === undefined) continue + const uniqueName = option === true ? `${name}_${sqlName(column)}_unique` : option + checkIdentifier(problems, `${name}.${column} unique`, uniqueName) + uniqueEntities.push({ kind: "unique", table: name, name: uniqueName, columns: [sqlName(column)] }) + } + + const tableEntity: SqliteTableEntity = { + kind: "table", + name, + primaryKey: definition.primaryKey === undefined ? null : { columns: pkColumns.filter(has).map(sqlName) }, + } + const defaults = inputs.filter((_, position) => columnEntities[position]!.default !== null).map(([column]) => column) + return { + _tag: "Table", + name, + columns: types, + ...(definition.tenantColumn !== undefined ? { tenantColumn: definition.tenantColumn } : undefined), + ...(defaults.length > 0 ? { defaults: defaults as unknown as Array> } : undefined), + ddl: { dialect: "sqlite", table: tableEntity, columns: columnEntities, indexes: indexEntities, uniques: uniqueEntities }, + problems, + } +} + +/** + * The entities of a set of SQLite tables, validated as one schema: table and + * index names do not collide (SQLite keeps them in one namespace). Fails with + * every problem found. + */ +export const sqliteEntitiesOf = ( + tables: ReadonlyArray>, +): Effect.Effect, SchemaDefinitionError> => { + const problems: ProblemSink = [] + const entities: Array = [] + const names = new Map() + const claim = (name: string, what: string) => { + const other = names.get(name.toLowerCase()) + if (other !== undefined) { + problems.push({ object: name, message: `names both ${other} and ${what}; SQLite keeps them in one namespace` }) + } + names.set(name.toLowerCase(), what) + } + for (const table of tables) { + if (!("ddl" in table) || (table.ddl as { dialect?: string }).dialect !== "sqlite") { + problems.push({ object: table.name, message: "is not a SQLite table with DDL; declare it with SQLite.table(name, { columns })" }) + continue + } + problems.push(...table.problems) + const { ddl } = table + claim(ddl.table.name, `table ${ddl.table.name}`) + for (const index of ddl.indexes) claim(index.name, `an index on ${ddl.table.name}`) + const constraints = new Set() + for (const unique of ddl.uniques) { + if (constraints.has(unique.name)) problems.push({ object: `${ddl.table.name}.${unique.name}`, message: "defined twice" }) + constraints.add(unique.name) + } + entities.push(ddl.table, ...ddl.columns, ...ddl.indexes, ...ddl.uniques) + } + return problems.length > 0 ? Effect.fail(definitionError(problems)) : Effect.succeed(sortEntities(entities)) +} diff --git a/src/schema/sqlite-diff.ts b/src/schema/sqlite-diff.ts new file mode 100644 index 0000000..8a5542a --- /dev/null +++ b/src/schema/sqlite-diff.ts @@ -0,0 +1,216 @@ +// Diff two SQLite schemas into migration ops. +// +// Pure and offline, like `diffPgSchemas`, with the same data-loss rule: +// dropping a table or a column needs a `confirm_data_loss` hint. A change +// SQLite's ALTER TABLE can make becomes an ALTER TABLE; any other change to a +// table becomes one `rebuild_table` op that carries the whole new table (see +// `sqlite-ops.ts`). Renames are never guessed: a `confirm_rename` hint turns +// a drop plus an add into a RENAME. + +import { canonicalJson, entityKey } from "./entities" +import { hintKey, type DiffResult, type Hint, type UnsupportedChange } from "./diff" +import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" +import type { SqliteMigrationOp } from "./sqlite-ops" +import { sqliteIdent } from "./sqlite-render" + +const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonicalJson(b) + +const pick = (entities: ReadonlyArray, kind: K) => + entities.filter((e): e is Extract => e.kind === kind) + +// Renames first, so later ops use the new names. Index drops before column +// drops, so no dropped column is still indexed. Tables and columns exist +// before the indexes that need them, and a rebuilt table before its indexes. +const opOrder: Record = { + rename_table: 0, + rename_column: 1, + drop_index: 2, + drop_column: 3, + drop_table: 4, + create_table: 5, + add_column: 6, + rebuild_table: 7, + create_index: 8, +} + +export const diffSqliteSchemas = ( + original: ReadonlyArray, + next: ReadonlyArray, + hints: ReadonlyArray = [], +): DiffResult => { + const given = new Set(hints.map(hintKey)) + const ops: Array = [] + const missingHints: Array = [] + const unsupported: Array = [] + // Renames run first; everything after compares the renamed schema with the next one. + const prev = applyRenames(original, next, hints, ops, unsupported) + /** Push `op` once every hint it needs was given; otherwise report the missing ones. */ + const confirm = (needed: Hint | ReadonlyArray, op: SqliteMigrationOp): void => { + const missing = (Array.isArray(needed) ? needed : [needed]).filter((hint) => !given.has(hintKey(hint))) + if (missing.length === 0) ops.push(op) + else missingHints.push(...missing) + } + + const tablesOf = (entities: ReadonlyArray) => new Map(pick(entities, "table").map((t) => [t.name, t])) + const prevTables = tablesOf(prev) + const nextTables = tablesOf(next) + const partsOf = (list: ReadonlyArray, table: string) => + list.filter((e) => e.table === table) + const columns = (entities: ReadonlyArray, table: string) => + partsOf(pick(entities, "column"), table).sort((a, b) => a.position - b.position) + + for (const [name, table] of nextTables) { + if (!prevTables.has(name)) ops.push({ op: "create_table", table, columns: columns(next, name), uniques: partsOf(pick(next, "unique"), name) }) + } + for (const name of prevTables.keys()) { + if (!nextTables.has(name)) confirm({ type: "confirm_data_loss", kind: "table", entity: name }, { op: "drop_table", name }) + } + + for (const [name, after] of nextTables) { + const before = prevTables.get(name) + if (before === undefined) continue + diffTable( + { table: before, columns: columns(prev, name), uniques: partsOf(pick(prev, "unique"), name) }, + { table: after, columns: columns(next, name), uniques: partsOf(pick(next, "unique"), name) }, + ops, + confirm, + ) + } + const rebuilt = new Set(ops.flatMap((op) => (op.op === "rebuild_table" ? [op.table.name] : []))) + + const prevIndexes = new Map(pick(prev, "index").map((i) => [entityKey(i), i])) + const nextIndexes = new Map(pick(next, "index").map((i) => [entityKey(i), i])) + for (const [key, index] of nextIndexes) { + const old = prevIndexes.get(key) + // A rebuilt table lost its indexes with the old table; all of them are created again. + if (old !== undefined && same(old, index) && !rebuilt.has(index.table)) continue + if (old !== undefined && !rebuilt.has(index.table)) ops.push({ op: "drop_index", name: old.name }) + ops.push({ op: "create_index", index }) + } + for (const [key, index] of prevIndexes) { + // A dropped table takes its indexes with it. + if (!nextIndexes.has(key) && nextTables.has(index.table) && !rebuilt.has(index.table)) ops.push({ op: "drop_index", name: index.name }) + } + + return { + ops: ops + .map((op, i) => [op, i] as const) + .sort(([a, i], [b, j]) => opOrder[a.op] - opOrder[b.op] || i - j) + .map(([op]) => op), + missingHints, + unsupported, + } +} + +interface TableState { + readonly table: SqliteTableEntity + readonly columns: ReadonlyArray + readonly uniques: ReadonlyArray +} + +/** What a column is, apart from its place in the table. */ +const definition = (column: SqliteColumnEntity) => ({ type: column.type, notNull: column.notNull, default: column.default }) + +/** + * Can `ALTER TABLE ADD COLUMN` add it? Not to a key or a UNIQUE constraint, + * not NOT NULL without a default, and not with an expression default, which + * SQLite refuses there. + */ +const addable = (column: SqliteColumnEntity, after: TableState): boolean => + !(after.table.primaryKey?.columns ?? []).includes(column.name) && + !after.uniques.some((u) => u.columns.includes(column.name)) && + (!column.notNull || (column.default !== null && column.default.toUpperCase() !== "NULL")) && + (column.default === null || !/^\s*\(|^\s*current_(time|date|timestamp)\b/i.test(column.default)) + +/** Can `ALTER TABLE DROP COLUMN` drop it? Not from a key or a UNIQUE constraint. */ +const droppable = (name: string, before: TableState): boolean => + !(before.table.primaryKey?.columns ?? []).includes(name) && !before.uniques.some((u) => u.columns.includes(name)) + +/** + * One table's changes: ALTER TABLE where SQLite allows it, otherwise a + * rebuild. A rebuild copies every column the two tables share. + */ +const diffTable = ( + before: TableState, + after: TableState, + ops: Array, + confirm: (needed: Hint | ReadonlyArray, op: SqliteMigrationOp) => void, +): void => { + const name = after.table.name + const old = new Map(before.columns.map((c) => [c.name, c])) + const kept = new Set(after.columns.map((c) => c.name)) + const added = after.columns.filter((c) => !old.has(c.name)) + const dropped = before.columns.filter((c) => !kept.has(c.name)) + const changed = after.columns.filter((c) => old.has(c.name) && !same(definition(old.get(c.name)!), definition(c))) + const needsRebuild = + changed.length > 0 || + !same(before.table.primaryKey, after.table.primaryKey) || + !same(byName(before.uniques), byName(after.uniques)) || + added.some((c) => !addable(c, after)) || + dropped.some((c) => !droppable(c.name, before)) + const dropHint = (column: string): Hint => ({ type: "confirm_data_loss", kind: "column", entity: `${name}.${column}` }) + if (needsRebuild) { + const rebuild: SqliteMigrationOp = { + op: "rebuild_table", + table: after.table, + columns: after.columns, + uniques: after.uniques, + copy: after.columns.filter((c) => old.has(c.name)).map((c) => c.name), + } + confirm(dropped.map((c) => dropHint(c.name)), rebuild) + return + } + for (const column of added) ops.push({ op: "add_column", column }) + for (const column of dropped) confirm(dropHint(column.name), { op: "drop_column", table: name, name: column.name }) +} + +const byName = (uniques: ReadonlyArray) => + [...uniques].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + +/** + * Apply the `confirm_rename` hints for tables and columns: an op for each, and + * `prev` as it reads after them. SQLite's RENAME rewrites the indexes and + * constraints that name the table or column, so they are renamed here too. + * A hint that does not match the two schemas is reported. + */ +const applyRenames = ( + prev: ReadonlyArray, + next: ReadonlyArray, + hints: ReadonlyArray, + ops: Array, + unsupported: Array, +): ReadonlyArray => { + let entities = prev + const has = (list: ReadonlyArray, kind: "table" | "column", table: string, name?: string) => + list.some((e) => (kind === "table" ? e.kind === "table" && e.name === table : e.kind === "column" && e.table === table && e.name === name)) + for (const hint of hints) { + if (hint.type !== "confirm_rename" || hint.kind !== "table") continue + if (!has(entities, "table", hint.entity) || has(entities, "table", hint.to) || !has(next, "table", hint.to) || has(next, "table", hint.entity)) { + unsupported.push({ entity: hint.entity, message: `cannot rename to ${hint.to}: the old table must exist only before, and the new one only after` }) + continue + } + ops.push({ op: "rename_table", from: hint.entity, to: hint.to }) + entities = entities.map((e) => (e.kind === "table" ? (e.name === hint.entity ? { ...e, name: hint.to } : e) : e.table === hint.entity ? { ...e, table: hint.to } : e)) + } + for (const hint of hints) { + if (hint.type !== "confirm_rename" || hint.kind !== "column") continue + const dot = hint.entity.lastIndexOf(".") + const table = hint.entity.slice(0, dot) + const from = hint.entity.slice(dot + 1) + if (dot <= 0 || !has(entities, "column", table, from) || has(entities, "column", table, hint.to) || !has(next, "column", table, hint.to) || has(next, "column", table, from)) { + unsupported.push({ entity: hint.entity, message: `cannot rename to ${hint.to}: the old column must exist only before, and the new one only after` }) + continue + } + ops.push({ op: "rename_column", table, from, to: hint.to }) + const rename = (name: string) => (name === from ? hint.to : name) + const quoted = sqliteIdent(from) + entities = entities.map((e): SqliteSchemaEntity => { + if (e.kind === "table") return e.name === table && e.primaryKey !== null ? { ...e, primaryKey: { columns: e.primaryKey.columns.map(rename) } } : e + if (e.table !== table) return e + if (e.kind === "column") return { ...e, name: rename(e.name) } + if (e.kind === "unique") return { ...e, columns: e.columns.map(rename) } + return { ...e, columns: e.columns.map((part) => part.split(quoted).join(sqliteIdent(hint.to))) } + }) + } + return entities +} diff --git a/src/schema/sqlite-drizzle.test.ts b/src/schema/sqlite-drizzle.test.ts new file mode 100644 index 0000000..5dabaa3 --- /dev/null +++ b/src/schema/sqlite-drizzle.test.ts @@ -0,0 +1,29 @@ +import { readdirSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { expect, it } from "vitest" +import { fromDrizzleSqliteSnapshot } from "./sqlite-drizzle" + +const fixture = join(import.meta.dirname, "..", "kit", "__fixtures__", "drizzle-sqlite") +const snapshot = () => { + const [folder] = readdirSync(fixture).filter((name) => /^\d{14}_/.test(name)) + return JSON.parse(readFileSync(join(fixture, folder!, "snapshot.json"), "utf8")) as { ddl: Array> } +} + +it("reads a drizzle-kit 1.0 primary key column as NOT NULL, and keeps UNIQUE constraints and index keys", () => { + const { entities, unsupported } = fromDrizzleSqliteSnapshot(snapshot()) + expect(unsupported).toEqual([]) + expect(entities).toContainEqual(expect.objectContaining({ kind: "column", table: "short_link", name: "id", notNull: true })) + expect(entities).toContainEqual({ kind: "unique", table: "membership", name: "membership_email_unique", columns: ["email"] }) + expect(entities).toContainEqual({ kind: "table", name: "membership", primaryKey: { columns: ["org_id", "user_id"] } }) + expect(entities).toContainEqual({ kind: "index", table: "membership", name: "membership_user_role", unique: false, columns: ['"user_id"', '"role"'] }) +}) + +it("reports what the entity model cannot hold", () => { + const json = snapshot() + json.ddl.push( + { entityType: "fks", table: "membership", name: "membership_org_fk" }, + { entityType: "columns", table: "membership", name: "n", type: "integer", notNull: true, autoincrement: true, default: null, generated: null }, + ) + expect(fromDrizzleSqliteSnapshot(json).unsupported).toEqual(["fks membership.membership_org_fk", "columns membership.n: AUTOINCREMENT"]) + expect(fromDrizzleSqliteSnapshot({ ...json, dialect: "postgresql" }).unsupported).toContain("dialect postgresql: only SQLite snapshots convert here") +}) diff --git a/src/schema/sqlite-drizzle.ts b/src/schema/sqlite-drizzle.ts new file mode 100644 index 0000000..8544277 --- /dev/null +++ b/src/schema/sqlite-drizzle.ts @@ -0,0 +1,101 @@ +// Reading a drizzle-kit 1.0 SQLite snapshot (`snapshot.json`, version 7) as +// entities, so `generate --baseline --from-drizzle` can adopt a drizzle-kit +// folder the way it does for Postgres. +// +// Only what the entity model holds converts: tables, columns, primary keys, +// indexes and UNIQUE constraints. Anything else (foreign keys, checks, views, +// generated or AUTOINCREMENT columns, partial indexes) is reported, and nothing +// is written. +// +// One reading is deliberate. drizzle-kit 1.0 treats a primary key column as +// not null and leaves `notNull: false` (and the NOT NULL keyword) out of its +// output. It is read as not null here, which is what effect-orm declares and +// writes for a key column. A table drizzle-kit created then still accepts a +// NULL in a single text key column until the table is rebuilt. + +import { sortEntities } from "./entities" +import { canonicalSqliteType, type SqliteSchemaEntity, type SqliteTableEntity } from "./sqlite-entities" +import { sqliteIdent } from "./sqlite-render" + +export interface DrizzleSqliteImport { + readonly entities: ReadonlyArray + /** Objects the snapshot holds that have no entity here. Empty when the import is complete. */ + readonly unsupported: ReadonlyArray +} + +type Json = Record + +const isRecord = (value: unknown): value is Json => typeof value === "object" && value !== null && !Array.isArray(value) +const str = (value: unknown): string => (typeof value === "string" ? value : String(value)) +const strings = (value: unknown): ReadonlyArray => (Array.isArray(value) ? value.map(str) : []) + +const indexPart = (part: unknown): string => { + if (typeof part === "string") return sqliteIdent(part) + if (!isRecord(part)) return str(part) + return part.isExpression === true ? str(part.value) : sqliteIdent(str(part.value)) +} + +/** Convert a drizzle-kit 1.0 SQLite snapshot. */ +export const fromDrizzleSqliteSnapshot = (json: unknown): DrizzleSqliteImport => { + const unsupported: Array = [] + if (!isRecord(json) || !Array.isArray(json.ddl)) { + return { entities: [], unsupported: ["not a drizzle-kit snapshot: it has no ddl list"] } + } + if (json.dialect !== "sqlite" && json.dialect !== "turso") { + unsupported.push(`dialect ${str(json.dialect)}: only SQLite snapshots convert here`) + } + const tables = new Map() + const entities: Array = [] + const positions = new Map() + for (const raw of json.ddl as ReadonlyArray) { + if (!isRecord(raw)) continue + const type = str(raw.entityType) + const where = `${type} ${raw.table !== undefined ? `${str(raw.table)}.` : ""}${str(raw.name)}` + switch (type) { + case "tables": + tables.set(str(raw.name), { kind: "table", name: str(raw.name), primaryKey: null }) + break + case "columns": { + const table = str(raw.table) + if (raw.generated !== null && raw.generated !== undefined) unsupported.push(`${where}: a generated column`) + if (raw.autoincrement === true) unsupported.push(`${where}: AUTOINCREMENT`) + const position = positions.get(table) ?? 0 + positions.set(table, position + 1) + entities.push({ + kind: "column", + table, + name: str(raw.name), + position, + type: canonicalSqliteType(str(raw.type)), + notNull: raw.notNull === true, + default: raw.default === null || raw.default === undefined ? null : str(raw.default), + }) + break + } + case "pks": { + const table = tables.get(str(raw.table)) + if (table === undefined) unsupported.push(`${where}: its table is not in the snapshot`) + else table.primaryKey = { columns: strings(raw.columns) } + break + } + case "indexes": + if (typeof raw.where === "string" && raw.where.length > 0) unsupported.push(`${where}: a partial index`) + entities.push({ + kind: "index", + table: str(raw.table), + name: str(raw.name), + unique: raw.isUnique === true, + columns: Array.isArray(raw.columns) ? raw.columns.map(indexPart) : [], + }) + break + case "uniques": + entities.push({ kind: "unique", table: str(raw.table), name: str(raw.name), columns: strings(raw.columns) }) + break + default: + unsupported.push(where) + } + } + const keyColumns = new Set([...tables.values()].flatMap((t) => (t.primaryKey?.columns ?? []).map((c) => `${t.name}.${c}`))) + const columns = entities.map((e) => (e.kind === "column" && keyColumns.has(`${e.table}.${e.name}`) ? { ...e, notNull: true } : e)) + return { entities: sortEntities([...tables.values(), ...columns]), unsupported } +} diff --git a/src/schema/sqlite-entities.ts b/src/schema/sqlite-entities.ts new file mode 100644 index 0000000..9d2493f --- /dev/null +++ b/src/schema/sqlite-entities.ts @@ -0,0 +1,63 @@ +// SQLite schema entities: the normalized, serializable form of a SQLite +// schema, as `entities.ts` is for ClickHouse and `pg-entities.ts` for Postgres. +// +// Types are stored as the declared type, lowercase (`text`, `integer`), which +// is what `PRAGMA table_info` reports back and what drizzle-kit writes. +// Expressions (defaults, index key parts) are SQL as written. + +import { Schema } from "effect" + +export const SqliteTableEntity = Schema.Struct({ + kind: Schema.Literal("table"), + name: Schema.String, + /** + * The primary key columns, or `null` for a table without one. SQLite keeps + * no name for the constraint, so none is stored. + */ + primaryKey: Schema.NullOr(Schema.Struct({ columns: Schema.Array(Schema.String) })), +}) +export type SqliteTableEntity = typeof SqliteTableEntity.Type + +export const SqliteColumnEntity = Schema.Struct({ + kind: Schema.Literal("column"), + table: Schema.String, + name: Schema.String, + /** Declaration order, the order `CREATE TABLE` writes the columns in. */ + position: Schema.Number, + /** The declared type, lowercase: `text`, `integer`, `real`, `blob`, `numeric`. */ + type: Schema.String, + notNull: Schema.Boolean, + /** The `DEFAULT` value as SQL: a literal, or an expression in parentheses. `null` for none. */ + default: Schema.NullOr(Schema.String), +}) +export type SqliteColumnEntity = typeof SqliteColumnEntity.Type + +/** An index, `CREATE [UNIQUE] INDEX`. */ +export const SqliteIndexEntity = Schema.Struct({ + kind: Schema.Literal("index"), + table: Schema.String, + name: Schema.String, + unique: Schema.Boolean, + /** Key parts as SQL: a quoted column (`"org_id"`) or an expression. */ + columns: Schema.Array(Schema.String), +}) +export type SqliteIndexEntity = typeof SqliteIndexEntity.Type + +/** + * A `UNIQUE` constraint, written in `CREATE TABLE` as drizzle-kit 1.0 writes + * `.unique()`. SQLite backs it with an automatic index and keeps no name for + * it, so `name` lives only here. Changing one rebuilds the table. + */ +export const SqliteUniqueEntity = Schema.Struct({ + kind: Schema.Literal("unique"), + table: Schema.String, + name: Schema.String, + columns: Schema.Array(Schema.String), +}) +export type SqliteUniqueEntity = typeof SqliteUniqueEntity.Type + +export const SqliteSchemaEntity = Schema.Union([SqliteTableEntity, SqliteColumnEntity, SqliteIndexEntity, SqliteUniqueEntity]) +export type SqliteSchemaEntity = typeof SqliteSchemaEntity.Type + +/** The declared type of a column, as `PRAGMA table_info` reports it back. */ +export const canonicalSqliteType = (type: string): string => type.trim().replace(/\s+/g, " ").toLowerCase() diff --git a/src/schema/sqlite-ops.ts b/src/schema/sqlite-ops.ts new file mode 100644 index 0000000..241f861 --- /dev/null +++ b/src/schema/sqlite-ops.ts @@ -0,0 +1,96 @@ +// SQLite migration operations and the SQL they render to. +// +// SQLite's ALTER TABLE can rename a table or a column, add a column and drop +// one, and little else. Any other change (a column's type, nullability or +// default, the primary key, a UNIQUE constraint) is a table rebuild: the +// procedure https://www.sqlite.org/lang_altertable.html#otheralter describes, +// as drizzle-kit writes it. Create the new table under a temporary name, copy +// the rows, drop the old table, rename the new one into place, and create its +// indexes again (dropping the table dropped them). + +import { Schema } from "effect" +import { SqliteColumnEntity, SqliteIndexEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" +import { renderSqliteColumnDefinition, renderSqliteCreateIndex, renderSqliteCreateTable, sqliteIdent } from "./sqlite-render" + +export const SqliteMigrationOp = Schema.Union([ + Schema.Struct({ + op: Schema.Literal("create_table"), + table: SqliteTableEntity, + columns: Schema.Array(SqliteColumnEntity), + uniques: Schema.Array(SqliteUniqueEntity), + }), + Schema.Struct({ op: Schema.Literal("drop_table"), name: Schema.String }), + Schema.Struct({ op: Schema.Literal("rename_table"), from: Schema.String, to: Schema.String }), + Schema.Struct({ op: Schema.Literal("add_column"), column: SqliteColumnEntity }), + Schema.Struct({ op: Schema.Literal("drop_column"), table: Schema.String, name: Schema.String }), + Schema.Struct({ op: Schema.Literal("rename_column"), table: Schema.String, from: Schema.String, to: Schema.String }), + /** The table as it will be, and the columns whose rows carry over (by their name in both tables). */ + Schema.Struct({ + op: Schema.Literal("rebuild_table"), + table: SqliteTableEntity, + columns: Schema.Array(SqliteColumnEntity), + uniques: Schema.Array(SqliteUniqueEntity), + copy: Schema.Array(Schema.String), + }), + Schema.Struct({ op: Schema.Literal("create_index"), index: SqliteIndexEntity }), + Schema.Struct({ op: Schema.Literal("drop_index"), name: Schema.String }), +]) +export type SqliteMigrationOp = typeof SqliteMigrationOp.Type + +/** Labels the plan prints. `rebuild`: the table is copied into a new one. */ +export type SqliteOpLabel = "metadata" | "destructive" | "rebuild" + +export const labelOfSqlite = (op: SqliteMigrationOp): SqliteOpLabel => { + switch (op.op) { + case "drop_table": + case "drop_column": + return "destructive" + case "rebuild_table": + return "rebuild" + default: + return "metadata" + } +} + +/** The temporary name a rebuilt table is created under, drizzle-kit's. */ +export const rebuildName = (table: string): string => `__new_${table}` + +const list = (names: ReadonlyArray): string => names.map(sqliteIdent).join(", ") + +/** The statements one op runs, in order. */ +export const renderSqliteOp = (op: SqliteMigrationOp): ReadonlyArray => { + switch (op.op) { + case "create_table": + return [renderSqliteCreateTable(op.table, op.columns, op.uniques)] + case "drop_table": + return [`DROP TABLE IF EXISTS ${sqliteIdent(op.name)}`] + case "rename_table": + return [`ALTER TABLE ${sqliteIdent(op.from)} RENAME TO ${sqliteIdent(op.to)}`] + case "add_column": + return [`ALTER TABLE ${sqliteIdent(op.column.table)} ADD COLUMN ${renderSqliteColumnDefinition(op.column)}`] + case "drop_column": + return [`ALTER TABLE ${sqliteIdent(op.table)} DROP COLUMN ${sqliteIdent(op.name)}`] + case "rename_column": + return [`ALTER TABLE ${sqliteIdent(op.table)} RENAME COLUMN ${sqliteIdent(op.from)} TO ${sqliteIdent(op.to)}`] + case "rebuild_table": { + const temporary = rebuildName(op.table.name) + const created = renderSqliteCreateTable({ ...op.table, name: temporary }, op.columns.map((c) => ({ ...c, table: temporary })), op.uniques, { ifNotExists: false }) + return [ + "PRAGMA foreign_keys=OFF", + created, + ...(op.copy.length === 0 ? [] : [`INSERT INTO ${sqliteIdent(temporary)}(${list(op.copy)}) SELECT ${list(op.copy)} FROM ${sqliteIdent(op.table.name)}`]), + `DROP TABLE ${sqliteIdent(op.table.name)}`, + `ALTER TABLE ${sqliteIdent(temporary)} RENAME TO ${sqliteIdent(op.table.name)}`, + "PRAGMA foreign_keys=ON", + ] + } + case "create_index": + return [renderSqliteCreateIndex(op.index)] + case "drop_index": + return [`DROP INDEX IF EXISTS ${sqliteIdent(op.name)}`] + } +} + +/** A migration.sql: each op's statements under a comment naming its label, one statement per breakpoint. */ +export const renderSqliteMigrationSql = (ops: ReadonlyArray, breakpoint: string): string => + `${ops.flatMap((op) => renderSqliteOp(op).map((statement) => `-- ${labelOfSqlite(op)}\n${statement};`)).join(`\n${breakpoint}\n`)}\n` diff --git a/src/schema/sqlite-render.ts b/src/schema/sqlite-render.ts new file mode 100644 index 0000000..f0279eb --- /dev/null +++ b/src/schema/sqlite-render.ts @@ -0,0 +1,60 @@ +// SQLite DDL rendering from entities: `CREATE TABLE` and `CREATE [UNIQUE] INDEX`. + +import { sqliteDialect } from "../sqlite/dialect" +import type { SqliteColumnEntity, SqliteIndexEntity, SqliteSchemaEntity, SqliteTableEntity, SqliteUniqueEntity } from "./sqlite-entities" + +/** An identifier, always double-quoted, so mixed case and keywords survive. */ +export const sqliteIdent = (name: string): string => sqliteDialect.quoteIdent(name) + +const list = (names: ReadonlyArray): string => names.map(sqliteIdent).join(", ") + +/** + * One column's definition. `primaryKey` writes the single-column key inline and + * `unique` a one-column `UNIQUE` constraint, as drizzle-kit 1.0 does. + */ +export const renderSqliteColumnDefinition = (column: SqliteColumnEntity, primaryKey = false, unique = false): string => { + const parts = [sqliteIdent(column.name), column.type] + if (primaryKey) parts.push("PRIMARY KEY") + if (unique) parts.push("UNIQUE") + if (column.notNull) parts.push("NOT NULL") + if (column.default !== null) parts.push(`DEFAULT ${column.default}`) + return parts.join(" ") +} + +export const renderSqliteCreateTable = ( + table: SqliteTableEntity, + columns: ReadonlyArray, + uniques: ReadonlyArray = [], + options: { readonly ifNotExists?: boolean } = {}, +): string => { + const key = table.primaryKey?.columns ?? [] + const inline = key.length === 1 ? key[0] : undefined + const inlineUnique = new Set(uniques.filter((u) => u.columns.length === 1).map((u) => u.columns[0])) + const body = [...columns] + .sort((a, b) => a.position - b.position) + .map((column) => renderSqliteColumnDefinition(column, column.name === inline, inlineUnique.has(column.name))) + if (key.length > 1) body.push(`PRIMARY KEY (${list(key)})`) + for (const unique of uniques) if (unique.columns.length > 1) body.push(`UNIQUE (${list(unique.columns)})`) + return `CREATE TABLE ${options.ifNotExists === false ? "" : "IF NOT EXISTS "}${sqliteIdent(table.name)} (\n\t${body.join(",\n\t")}\n)` +} + +export const renderSqliteCreateIndex = (index: SqliteIndexEntity): string => + `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${sqliteIdent(index.name)} ON ${sqliteIdent(index.table)} (${index.columns.join(", ")})` + +/** Every CREATE statement for a schema: tables, then indexes. */ +export const renderSqliteSchema = (entities: ReadonlyArray): ReadonlyArray => { + const columns = entities.filter((e): e is SqliteColumnEntity => e.kind === "column") + const uniques = entities.filter((e): e is SqliteUniqueEntity => e.kind === "unique") + return [ + ...entities + .filter((e): e is SqliteTableEntity => e.kind === "table") + .map((table) => + renderSqliteCreateTable( + table, + columns.filter((c) => c.table === table.name), + uniques.filter((u) => u.table === table.name), + ), + ), + ...entities.filter((e): e is SqliteIndexEntity => e.kind === "index").map(renderSqliteCreateIndex), + ] +} diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index bf0fe19..89106c7 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -11,6 +11,8 @@ import type { DateTime } from "effect" export interface SqlSyntax { /** The dialect's name (`clickhouse`, `postgres`, …), for the few choices expression code makes per dialect. */ readonly dialect: string + /** False when the dialect has no ILIKE operator. */ + readonly ilike?: boolean /** One identifier (a column, alias, table or schema name), quoted as needed. */ readonly quoteIdent: (name: string) => string /** A string as a quoted literal. */ diff --git a/src/sqlite.ts b/src/sqlite.ts new file mode 100644 index 0000000..de5b1f7 --- /dev/null +++ b/src/sqlite.ts @@ -0,0 +1,88 @@ +// @maple-dev/effect-orm/sqlite +// +// SQLite and Cloudflare D1 from one import: column types, tables with their +// DDL, SQLite functions, the query builder and a `compile` that defaults to +// the SQLite dialect. Migrations stay with the caller for now. See +// docs/sqlite.md. +import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" +import { compiled, compiledUnsafe } from "./ch/failure" +import { table as queryTable, type Table, type TableOptions } from "./ch/table" +import type { ColumnDefs } from "./ch/types" +import { + externalSqliteTable, + isTableDefinition, + sqliteTable, + type ColumnInput, + type ColumnsOf, + type DefaultedColumnsOf, + type ExternalTableDefinition, + type SqliteSchemaTable, + type TableDefinition, +} from "./schema/sqlite-define" +import { sqliteDialect } from "./sqlite/dialect" + +export * from "./core" +export * from "./sqlite/types" +export * from "./sqlite/functions" +export type { TableOptions } +export { + column, + index, + uniqueIndex, + type ColumnInput, + type ColumnOptions, + type ColumnSpec, + type ColumnsOf, + type DefaultedColumnsOf, + type ExternalTableDefinition, + type IndexSpec, + type SqliteSchemaTable, + type TableDdl, + type TableDefinition, +} from "./schema/sqlite-define" +export { SchemaDefinitionError, type DefinitionProblem, type DdlExpr, type DdlKey } from "./schema/define" + +/** + * A SQLite table. With `{ columns, primaryKey, indexes }` it carries its DDL + * (`renderSqliteSchema` from `/schema` writes it). With `{ external: true, columns }`, + * or a bare record of columns and query options, it is a query schema only. + */ +export function table>( + name: Name, + definition: ExternalTableDefinition, +): Table, DefaultedColumnsOf, never> +export function table>( + name: Name, + definition: TableDefinition, +): SqliteSchemaTable, DefaultedColumnsOf> +export function table< + const Name extends string, + const Columns extends ColumnDefs, + const Defaulted extends keyof Columns & string = never, + const Computed extends keyof Columns & string = never, +>(name: Name, columns: Columns, options?: TableOptions): Table +export function table(name: string, definition: any, options?: TableOptions): Table { + if (options !== undefined || !isTableDefinition(definition)) return queryTable(name, definition, options) + if ("external" in definition && definition.external === true) return externalSqliteTable(name, definition as ExternalTableDefinition) + return sqliteTable(name, definition as TableDefinition) +} +export { sqliteDialect, d1Dialect } from "./sqlite/dialect" + +/** The shared compiler, defaulting to SQLite. Pass d1Dialect for D1's limits. */ +export const compile: typeof compileCH = ( + query: Parameters[0], + params?: Record, + options?: Parameters[2], +) => compiled(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) + +/** Compile immediately, throwing on a failed compile. */ +export const compileUnsafe: typeof compileCHUnsafe = ( + query: Parameters[0], + params?: Record, + options?: Parameters[2], +) => compiledUnsafe(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) + +export const compileUnion: typeof compileUnionCH = (query, params, options) => + compileUnionCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) +export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (query, params, options) => + compileUnionUnsafeCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) diff --git a/src/sqlite/d1.test.ts b/src/sqlite/d1.test.ts new file mode 100644 index 0000000..ff9c123 --- /dev/null +++ b/src/sqlite/d1.test.ts @@ -0,0 +1,88 @@ +// A real local Workers D1 binding; no cloud account or network database needed. +import * as D1Client from "@effect/sql-d1/D1Client" +import { Cause, Effect, Exit, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { Miniflare } from "miniflare" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Ack = SQLite.table("whats_new_ack", { + id: SQLite.text, + entry_id: SQLite.text, + user_id: SQLite.text, + org_id: SQLite.text, + acked_at: SQLite.text, +}, { tenantColumn: "org_id" }) +const first = { id: "ack_1", entry_id: "release_1", user_id: "user_1", org_id: "org_1", acked_at: "2026-10-01T09:00:00.000Z" } +const second = { ...first, id: "ack_2", entry_id: "release_2" } +const other = { ...first, id: "ack_3", user_id: "user_2", org_id: "org_2" } + +const withD1 = async (body: (db: Db.DatabaseApi, statements: Array) => Effect.Effect) => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('D1 dialect test') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + try { + const binding = await mf.getD1Database("DB") + return await Effect.runPromise(Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + const statements: Array = [] + const db = Db.fromSqlClient(client, { + dialect: SQLite.d1Dialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* db.execute({ sql: "CREATE TABLE whats_new_ack (id TEXT PRIMARY KEY, entry_id TEXT NOT NULL, user_id TEXT NOT NULL, org_id TEXT NOT NULL, acked_at TEXT NOT NULL, UNIQUE(entry_id, user_id))" }) + statements.length = 0 + return yield* body(db, statements) + }).pipe(Effect.provide(D1Client.layer({ db: binding })))) + } finally { + await mf.dispose() + } +} + +it("copies acknowledgments on D1, keeps the first row, scopes reads and pages explicitly across tenants", async () => { + const result = await withD1((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Ack).values([first, second, other]).onConflictDoNothing().crossTenant()) + yield* db.run(SQLite.insertInto(Ack).values({ ...first, id: "ack_9", acked_at: "2026-10-09T00:00:00.000Z" }).onConflictDoNothing()) + const byUser = yield* db.run(SQLite.from(Ack).select("entry_id").where(($) => [$.org_id.eq("org_1"), $.user_id.eq("user_1")]).orderBy(["entry_id", "asc"])) + const page = yield* db.run(SQLite.from(Ack).select().where(($) => [$.id.gt("ack_1")]).orderBy(["id", "asc"]).limit(2).crossTenant()) + const kept = yield* db.run(SQLite.from(Ack).select().where(($) => [$.org_id.eq("org_1"), $.id.eq("ack_1")])) + return { byUser, page, kept } + })) + expect(result).toEqual({ byUser: [{ entry_id: "release_1" }, { entry_id: "release_2" }], page: [second, other], kept: [first] }) +}, 30_000) + +it("refuses an unscoped query, too many bindings and interactive transactions before dispatch", async () => { + const seen = await withD1((db, statements) => Effect.gen(function* () { + const scope = yield* Effect.exit(db.run(SQLite.from(Ack).select())) + const limits = yield* Effect.exit(db.run(SQLite.insertInto(Ack).values(Array.from({ length: 21 }, (_, i) => ({ ...first, id: `${i}` }))).crossTenant())) + let bodyRan = false + const transaction = yield* Effect.exit(db.transaction(Effect.sync(() => { bodyRan = true }))) + return { scope, limits, transaction, bodyRan, statements: [...statements] } + })) + if (!Exit.isFailure(seen.scope) || !Exit.isFailure(seen.limits) || !Exit.isFailure(seen.transaction)) throw new Error("a D1 guard accepted an invalid request") + expect(Cause.findDefect(seen.scope.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/TenantScopeViolation" } }) + expect(Cause.findError(seen.limits.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/QueryBuilderError" } }) + expect(Cause.findError(seen.transaction.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/TransactionUnsupported" } }) + expect(seen.bodyRan).toBe(false) + expect(seen.statements).toEqual([]) +}, 30_000) + +it("decodes D1 JSON and blob wire values and returns typed errors for malformed rows", async () => { + const Payloads = SQLite.table("payloads", { id: SQLite.integer, payload: SQLite.json(Schema.Struct({ ok: Schema.Boolean })), bytes: SQLite.blob }) + const seen = await withD1((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE payloads (id INTEGER, payload TEXT, bytes BLOB)" }) + yield* db.run(SQLite.insertInto(Payloads).values({ id: 1, payload: { ok: true }, bytes: new Uint8Array([0, 127, 255]) })) + const rows = yield* db.run(SQLite.from(Payloads).select()) + yield* db.execute({ sql: "UPDATE payloads SET payload = '{}' WHERE id = 1" }) + const malformed = yield* Effect.exit(db.run(SQLite.from(Payloads).select())) + return { rows, malformed } + })) + expect(seen.rows).toEqual([{ id: 1, payload: { ok: true }, bytes: new Uint8Array([0, 127, 255]) }]) + if (!Exit.isFailure(seen.malformed)) throw new Error("malformed JSON was accepted") + expect(Cause.findError(seen.malformed.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) +}, 30_000) diff --git a/src/sqlite/ddl.test.ts b/src/sqlite/ddl.test.ts new file mode 100644 index 0000000..cdde870 --- /dev/null +++ b/src/sqlite/ddl.test.ts @@ -0,0 +1,185 @@ +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { expect, it } from "vitest" +import * as Db from "../database" +import { canonicalJson, renderSqliteSchema, SqliteSchemaEntity, sqliteEntitiesOf } from "../schema" +import * as SQLite from "../sqlite" +import { shapeAfter } from "./shape.test-support" + +// drizzle-kit 0.31.11 (drizzle-orm 0.45.4) `generate` output for the same table. +const drizzleShortLink = `CREATE TABLE \`short_link\` ( + \`id\` text PRIMARY KEY NOT NULL, + \`code\` text NOT NULL, + \`target_url\` text NOT NULL, + \`target_hash\` text NOT NULL, + \`org_id\` text, + \`created_by\` text, + \`subject_type\` text NOT NULL, + \`subject_id\` text NOT NULL, + \`purpose\` text NOT NULL, + \`prospect_id\` text, + \`click_count\` integer NOT NULL, + \`first_accessed_at\` text, + \`last_accessed_at\` text, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX \`short_link_code\` ON \`short_link\` (\`code\`);--> statement-breakpoint +CREATE INDEX \`short_link_org\` ON \`short_link\` (\`org_id\`);` + +const named = [0]>(type: T, name: string) => SQLite.column(type, { name }) +const optionalText = (name: string) => named(SQLite.nullable(SQLite.text), name) + +const ShortLink = SQLite.table("short_link", { + columns: { + id: SQLite.text, + code: SQLite.text, + targetUrl: named(SQLite.text, "target_url"), + targetHash: named(SQLite.text, "target_hash"), + orgId: optionalText("org_id"), + createdBy: optionalText("created_by"), + subjectType: named(SQLite.text, "subject_type"), + subjectId: named(SQLite.text, "subject_id"), + purpose: SQLite.text, + prospectId: optionalText("prospect_id"), + clickCount: named(SQLite.integer, "click_count"), + firstAccessedAt: optionalText("first_accessed_at"), + lastAccessedAt: optionalText("last_accessed_at"), + createdAt: named(SQLite.text, "created_at"), + updatedAt: named(SQLite.text, "updated_at"), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("short_link_code", ["code"]), SQLite.index("short_link_org", ["orgId"])], + tenantColumn: "orgId", +}) + +const drizzleStatements = (sql: string) => sql.split("--> statement-breakpoint").map((s) => s.trim()).filter((s) => s.length > 0) + +const ddlOf = (tables: Parameters[0]) => Effect.runPromise(Effect.map(sqliteEntitiesOf(tables), renderSqliteSchema)) + +it("creates short_link exactly as drizzle-kit's SQL does, by PRAGMA table_info and index_list", async () => { + const ours = await shapeAfter(await ddlOf([ShortLink])) + expect(ours).toEqual(await shapeAfter(drizzleStatements(drizzleShortLink))) + expect(Object.keys(ours)).toEqual(["short_link"]) +}) + +// drizzle-kit 1.0.0-rc.5 for: composite primaryKey, .default(...) literals, sql`(CURRENT_TIMESTAMP)`, +// text("email").unique(), real, blob, numeric, a two-column index. drizzle-kit 1.0 writes +// .unique() inline, as a UNIQUE constraint; 0.31 wrote a separate unique index. +const drizzleMembership = `CREATE TABLE \`membership\` ( + \`org_id\` text NOT NULL, + \`user_id\` text NOT NULL, + \`role\` text DEFAULT 'member' NOT NULL, + \`weight\` real DEFAULT 1.5 NOT NULL, + \`active\` integer DEFAULT true NOT NULL, + \`avatar\` blob, + \`amount\` numeric, + \`email\` text UNIQUE, + \`created_at\` text DEFAULT (CURRENT_TIMESTAMP) NOT NULL, + CONSTRAINT \`membership_pk\` PRIMARY KEY(\`org_id\`, \`user_id\`) +); +--> statement-breakpoint +CREATE INDEX \`membership_user_role\` ON \`membership\` (\`user_id\`,\`role\`);` + +// drizzle-kit 1.0.0-rc.5 for short_link. It drops NOT NULL from a primary key column even when +// the column says .notNull(), so SQLite accepts a NULL id. We keep NOT NULL. +const drizzleV1ShortLinkTable = drizzleShortLink + .replace("`id` text PRIMARY KEY NOT NULL", "`id` text PRIMARY KEY") + .split("--> statement-breakpoint")[0]! + +const Membership = SQLite.table("membership", { + columns: { + orgId: named(SQLite.text, "org_id"), + userId: named(SQLite.text, "user_id"), + role: SQLite.column(SQLite.text, { default: "member" }), + weight: SQLite.column(SQLite.real, { default: 1.5 }), + active: SQLite.column(SQLite.boolean, { default: true }), + avatar: SQLite.nullable(SQLite.blob), + amount: SQLite.nullable(SQLite.custom("numeric", Schema.Finite)), + email: SQLite.column(SQLite.nullable(SQLite.text), { unique: true }), + createdAt: SQLite.column(SQLite.text, { name: "created_at", defaultExpr: "(CURRENT_TIMESTAMP)" }), + }, + primaryKey: ["orgId", "userId"], + indexes: [SQLite.index("membership_user_role", ["userId", "role"])], + tenantColumn: "orgId", +}) + +it("matches drizzle-kit 1.0 for short_link, except that the primary key column stays NOT NULL", async () => { + const ours = (await shapeAfter(await ddlOf([ShortLink]))) as { short_link: { columns: Array<{ name: string; notnull: number }> } } + const theirs = (await shapeAfter([drizzleV1ShortLinkTable, ...drizzleStatements(drizzleShortLink).slice(1)])) as typeof ours + expect(ours.short_link.columns.find((c) => c.name === "id")?.notnull).toBe(1) + expect(theirs.short_link.columns.find((c) => c.name === "id")?.notnull).toBe(0) + const withoutKeyNull = (shape: typeof ours) => ({ ...shape, short_link: { ...shape.short_link, columns: shape.short_link.columns.map((c) => (c.name === "id" ? { ...c, notnull: 1 } : c)) } }) + expect(withoutKeyNull(ours)).toEqual(withoutKeyNull(theirs)) +}) + +it("creates composite keys, defaults, unique columns and every storage class as drizzle-kit 1.0 does", async () => { + expect(await shapeAfter(await ddlOf([Membership]))).toEqual(await shapeAfter(drizzleStatements(drizzleMembership))) +}) + +it("lets an insert leave out defaulted columns, and SQLite fills them", async () => { + const rows = await Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + for (const sql of yield* Effect.map(sqliteEntitiesOf([Membership]), renderSqliteSchema)) yield* db.execute({ sql }) + yield* db.run(SQLite.insertInto(Membership).values({ orgId: "o", userId: "u", avatar: null, amount: null, email: null })) + return yield* db.run(SQLite.from(Membership).select("role", "weight", "active")) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + expect(rows).toEqual([{ role: "member", weight: 1.5, active: true }]) +}) + +it("renders DSL defaults and expression index keys that SQLite accepts", async () => { + const Codes = SQLite.table("codes", { + columns: { + id: SQLite.integer, + code: SQLite.text, + hits: SQLite.column(SQLite.integer, { defaultExpr: () => SQLite.lit(1) }), + }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("codes_code_lower", ($) => [SQLite.lower($.code)])], + }) + const statements = await ddlOf([Codes]) + expect(statements).toEqual([ + 'CREATE TABLE IF NOT EXISTS "codes" (\n\t"id" integer PRIMARY KEY NOT NULL,\n\t"code" text NOT NULL,\n\t"hits" integer NOT NULL DEFAULT (1)\n)', + 'CREATE UNIQUE INDEX IF NOT EXISTS "codes_code_lower" ON "codes" (lower("code"))', + ]) + const shape = (await shapeAfter(statements)) as { codes: { indexes: Array<{ keys: Array<{ cid: number }> }> } } + expect(shape.codes.indexes[0]?.keys[0]?.cid).toBe(-2) +}) + +it("keeps the query-only forms free of DDL", async () => { + const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer, orgId: named(SQLite.text, "org_id") }, tenantColumn: "orgId" }) + const Legacy = SQLite.table("legacy", { id: SQLite.integer, note: SQLite.text }, { defaults: ["note"] }) + expect("ddl" in External || "ddl" in Legacy).toBe(false) + expect(SQLite.compileUnsafe(SQLite.from(External).select("orgId").where(($) => [$.orgId.eq("o")])).sql).toContain('"ext"."org_id" AS "orgId"') + const exit = await Effect.runPromiseExit(sqliteEntitiesOf([External as never])) + expect(exit._tag).toBe("Failure") +}) + +it("records every definition problem, and sqliteEntitiesOf fails with all of them", async () => { + const Bad = SQLite.table("sqlite_bad", { + columns: { + id: SQLite.nullable(SQLite.text), + both: SQLite.column(SQLite.text, { default: "a", defaultExpr: "('b')" }), + }, + primaryKey: ["id", "missing" as "id"], + indexes: [SQLite.index("bad_none", ["nope" as "id"])], + }) + const Clash = SQLite.table("clash", { columns: { id: SQLite.text }, indexes: [SQLite.index("short_link", ["id"])] }) + const exit = await Effect.runPromiseExit(sqliteEntitiesOf([Bad, Clash, ShortLink])) + if (exit._tag !== "Failure") throw new Error("a broken schema was accepted") + const messages = JSON.stringify(exit.cause) + for (const expected of ["starts with sqlite_", "cannot be nullable", "missing is not a column", "one of default, defaultExpr", "nope is not a column", "one namespace"]) { + expect(messages).toContain(expected) + } +}) + +it("keeps entities as plain data that survive a JSON round trip, sorted the same every time", async () => { + const entities = await Effect.runPromise(sqliteEntitiesOf([Membership, ShortLink])) + const again = await Effect.runPromise(sqliteEntitiesOf([ShortLink, Membership])) + expect(canonicalJson(again)).toBe(canonicalJson(entities)) + const decoded = Schema.decodeUnknownSync(Schema.Array(SqliteSchemaEntity))(JSON.parse(canonicalJson(entities))) + expect(renderSqliteSchema(decoded)).toEqual(renderSqliteSchema(entities)) + expect(entities.map((e) => e.kind).filter((k, i, all) => all.indexOf(k) === i)).toEqual(["table", "column", "index", "unique"]) +}) diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts new file mode 100644 index 0000000..748e507 --- /dev/null +++ b/src/sqlite/dialect.ts @@ -0,0 +1,70 @@ +import { DateTime } from "effect" +import { noTransactions, type Dialect } from "../ch/dialect" +import { QueryBuilderError } from "../ch/errors" +import { fail } from "../ch/failure" +import { PARAM_MARKER_PREFIX } from "../ch/param" +import { PgTimestampLiteral, timestampLiteral } from "../pg/types" + +const hex = (value: Uint8Array): string => + Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("") + +/** Hex text keeps NUL and the compiler's reserved param marker out of SQL text. */ +const quoteString = (value: string): string => + value.includes(PARAM_MARKER_PREFIX) || value.includes("\0") + ? `CAST(X'${hex(new TextEncoder().encode(value))}' AS TEXT)` + : `'${value.replace(/'/g, "''")}'` + +const literal = (value: unknown, context: string): string => { + if (value === null) return "NULL" + if (typeof value === "string") return quoteString(value) + if (typeof value === "boolean") return value ? "1" : "0" + if (typeof value === "number" && Number.isFinite(value)) return String(value) + if (value instanceof Uint8Array) return `X'${hex(value)}'` + return fail(new QueryBuilderError({ code: "InvalidLiteral", message: `${context}: cannot write this value as a SQLite literal` }), "NULL") +} + +/** SQLite with positional bindings and transactions supplied by the caller's SqlClient. */ +export const sqliteDialect: Dialect = { + name: "sqlite", + functions: "sqlite", + ilike: false, + quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, + quoteString, + literal, + dateTimeLiteral: (value) => quoteString(DateTime.formatIso(value)), + params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766 }, + clauses: { + format: false, + derivedTableAlias: true, + groupByAlias: false, + parenthesizeUnionBranches: false, + wrapUnionBranches: true, + offsetRequiresLimit: true, + returning: true, + onConflict: true, + onConflictConstraint: false, + insertSelectNeedsWhere: true, + defaultInValues: false, + distinctOn: false, + }, + paramCodecs: { + dateTime: PgTimestampLiteral, + dateTimeSeconds: timestampLiteral((millis) => new Date(Math.floor(millis / 1000) * 1000).toISOString()), + }, + transactions: { + support: "full", + savepoints: true, + isolationLevels: ["serializable"], + accessModes: false, + deferrable: false, + setTransaction: () => undefined, + }, +} + +/** D1 executes SQLite SQL, limits each statement to 100 bindings, and has no interactive transactions. */ +export const d1Dialect: Dialect = { + ...sqliteDialect, + name: "d1", + params: { ...sqliteDialect.params, _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 100 }, + transactions: noTransactions, +} diff --git a/src/sqlite/functions.ts b/src/sqlite/functions.ts new file mode 100644 index 0000000..afcc9b6 --- /dev/null +++ b/src/sqlite/functions.ts @@ -0,0 +1,37 @@ +import { Schema } from "effect" +import { makeExpr, type Condition, type Expr } from "../ch/expr" +import { schemaOf } from "../ch/define-fn" +import { builtins } from "../ch/functions/builtin" +import { compile } from "../sql/sql-fragment" + +const aggregate = builtins("sqlite", "aggregate") +const scalar = builtins("sqlite", "scalar") +const sql = (expr: Expr | Condition): string => compile(expr.toFragment()) +const nullableNumber = Schema.NullOr(Schema.Finite) + +export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), Schema.Int) +export const countDistinct =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`, "count"), Schema.Int) +export const countIf =

(condition: Condition

): Expr => + makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`, "count"), Schema.Int) +export const sum =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`, "sum"), nullableNumber) +export const avg =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`, "avg"), nullableNumber) +export const min = (expr: Expr): Expr => { + const schema = schemaOf(expr) + return makeExpr(aggregate.lazy(() => `min(${sql(expr)})`, "min"), schema === undefined ? undefined : Schema.NullOr(schema)) +} +export const max = (expr: Expr): Expr => { + const schema = schemaOf(expr) + return makeExpr(aggregate.lazy(() => `max(${sql(expr)})`, "max"), schema === undefined ? undefined : Schema.NullOr(schema)) +} +export const lower =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `lower(${sql(expr)})`, "lower"), Schema.NullOr(Schema.String)) +export const upper =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `upper(${sql(expr)})`, "upper"), Schema.NullOr(Schema.String)) +/** Text length in SQLite characters, or blob length in bytes. */ +export const length =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `length(${sql(expr)})`, "length"), Schema.NullOr(Schema.Int)) +export const coalesce = (first: Expr, fallback: Expr): Expr => + makeExpr(scalar.lazy(() => `coalesce(${sql(first)}, ${sql(fallback)})`, "coalesce"), schemaOf(fallback)) diff --git a/src/sqlite/shape.test-support.ts b/src/sqlite/shape.test-support.ts new file mode 100644 index 0000000..12d5f69 --- /dev/null +++ b/src/sqlite/shape.test-support.ts @@ -0,0 +1,35 @@ +// Test support: what SQLite reports about a schema, to compare two ways of building it. +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as Db from "../database" +import { sqliteDialect } from "./dialect" + +/** What SQLite reports about every table and index after running `statements` on an empty database. */ +export const shapeAfter = (statements: ReadonlyArray) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: sqliteDialect }) + for (const sql of statements) yield* db.execute({ sql }) + const tables = yield* db.query({ sql: "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name" }) + const shape: Record = {} + for (const { name } of tables as ReadonlyArray<{ name: string }>) { + const columns = yield* db.query({ sql: `SELECT * FROM pragma_table_info('${name}')` }) + const list = yield* db.query({ sql: `SELECT name, "unique", origin, partial FROM pragma_index_list('${name}') ORDER BY name` }) + const indexes = [] + for (const index of list as ReadonlyArray<{ name: string }>) { + const keys = yield* db.query({ sql: `SELECT seqno, cid, name, "desc", coll, "key" FROM pragma_index_xinfo('${index.name}')` }) + indexes.push({ ...index, keys }) + } + shape[name] = { columns, indexes } + } + return shape + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + + +/** The rows `select` returns after running `statements` on an empty database. */ +export const rowsAfter = (statements: ReadonlyArray, select: string) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: sqliteDialect }) + for (const sql of statements) yield* db.execute({ sql }) + return yield* db.query({ sql: select }) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) diff --git a/src/sqlite/sqlite.test-d.ts b/src/sqlite/sqlite.test-d.ts new file mode 100644 index 0000000..f3cbecb --- /dev/null +++ b/src/sqlite/sqlite.test-d.ts @@ -0,0 +1,44 @@ +import type { Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as SQLite from "../sqlite" + +const Items = SQLite.table("items", { id: SQLite.integer, note: SQLite.text }) +const query = SQLite.from(Items).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +const compiled = SQLite.compileUnsafe(query, { id: 1 }) +expectTypeOf>>().toEqualTypeOf>() +const effect = SQLite.compile(query, { id: 1 }) +expectTypeOf>().toEqualTypeOf() +const insert = SQLite.compileUnsafe(SQLite.insertInto(Items).values({ id: 1, note: "n" }).returning("note")) +expectTypeOf>>().toEqualTypeOf>() +// @ts-expect-error a select cannot name a missing column +SQLite.from(Items).select("missing") +// @ts-expect-error required params remain required through the SQLite compile wrapper +SQLite.compile(query, {}) +// @ts-expect-error an integer column cannot take a string +SQLite.insertInto(Items).values({ id: "wrong", note: "n" }) +const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer } }) +SQLite.from(External).select("id") +// @ts-expect-error an external table has only its declared columns +SQLite.from(External).select("columns") + +const Links = SQLite.table("links", { + columns: { + id: SQLite.text, + orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }), + hits: SQLite.column(SQLite.integer, { default: 0 }), + code: SQLite.column(SQLite.text, { unique: true }), + }, + primaryKey: ["id"], + indexes: [SQLite.index("links_org", ["orgId"])], +}) +expectTypeOf(Links.ddl.dialect).toEqualTypeOf<"sqlite">() +// A defaulted column may be left out of an insert; a nullable one too. +SQLite.insertInto(Links).values({ id: "a", code: "c" }) +// @ts-expect-error a unique column without a default is still required +SQLite.insertInto(Links).values({ id: "a" }) +// @ts-expect-error a primary key names declared columns only +SQLite.table("bad", { columns: { id: SQLite.text }, primaryKey: ["nope"] }) +// @ts-expect-error a default is checked against the column's type +SQLite.table("bad", { columns: { n: SQLite.column(SQLite.integer, { default: "x" }) } }) +// @ts-expect-error an external table carries no DDL +External.ddl diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts new file mode 100644 index 0000000..eef78c0 --- /dev/null +++ b/src/sqlite/sqlite.test.ts @@ -0,0 +1,155 @@ +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Cause, DateTime, Effect, Exit, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Items = SQLite.table("items", { + id: SQLite.integer, + note: SQLite.text, + enabled: SQLite.boolean, + score: SQLite.nullable(SQLite.real), +}) +const withDatabase = (body: (db: Db.DatabaseApi) => Effect.Effect) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + yield* db.execute({ sql: "CREATE TABLE items (id INTEGER PRIMARY KEY, note TEXT NOT NULL, enabled INTEGER NOT NULL, score REAL DEFAULT 7)" }) + return yield* body(db) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + +it("runs and decodes SQLite writes and selects through an existing Effect SqlClient", async () => { + const seen = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "O'Reilly\\notes", enabled: true, score: null })) + return yield* db.run(SQLite.from(Items).select().where(($) => [$.enabled.eq(SQLite.param.bool("enabled"))]), { enabled: true }) + })) + expect(seen).toEqual([{ id: 1, note: "O'Reilly\\notes", enabled: true, score: null }]) +}) + +it("keeps query and union pagination when OFFSET has no LIMIT and branches have local limits", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([1, 2, 3].map((id) => ({ id, note: `${id}`, enabled: false, score: null })))) + const offset = yield* db.run(SQLite.from(Items).select("id").orderBy(["id", "asc"]).offset(1)) + const branch = (direction: "asc" | "desc") => SQLite.from(Items).select("id").orderBy(["id", direction]).limit(1) + const union = yield* db.run(SQLite.unionAll(branch("asc"), branch("desc")).orderBy(["id", "asc"]).offset(1)) + return { offset, union } + })) + expect(rows).toEqual({ offset: [{ id: 2 }, { id: 3 }], union: [{ id: 3 }] }) +}) + +it("refuses unsupported SQLite syntax while compiling, before sending SQL", () => { + const defaulted = SQLite.table("items", { id: SQLite.integer, note: SQLite.text }, { defaults: ["note"] }) + expect(() => SQLite.compileUnsafe(SQLite.insertInto(defaulted).values([{ id: 1, note: "a" }, { id: 2 }]))).toThrow(/DEFAULT/) + expect(() => SQLite.compileUnsafe(SQLite.insertInto(Items) + .values({ id: 1, note: "a", enabled: true, score: null }) + .onConflictDoNothing({ target: { constraint: "items_pkey" } }))).toThrow(/ON CONSTRAINT/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").distinctOn("id"))).toThrow(/DISTINCT ON/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.note.ilike("a%")]))).toThrow(/ILIKE/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").forUpdate())).toThrow(/row locks/) +}) + +it("runs INSERT SELECT with ON CONFLICT even when the source has no WHERE", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "first", enabled: true, score: null })) + yield* db.run(SQLite.insertInto(Items).select(SQLite.from(Items).select()).onConflictDoNothing()) + return yield* db.run(SQLite.from(Items).select("id")) + })) + expect(rows).toEqual([{ id: 1 }]) +}) + +it("decodes SQLite aggregates, including SQL NULL for an empty aggregate", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([ + { id: 1, note: "A", enabled: true, score: 2 }, + { id: 2, note: "B", enabled: false, score: 4 }, + ])) + const totals = yield* db.run(SQLite.from(Items).select(($) => ({ n: SQLite.count(), enabled: SQLite.countIf($.enabled.eq(true)), total: SQLite.sum($.score), avg: SQLite.avg($.score), min: SQLite.min($.score), max: SQLite.max($.score) }))) + const empty = yield* db.run(SQLite.from(Items).select(($) => ({ n: SQLite.count(), sum: SQLite.sum($.score) })).where(($) => [$.id.eq(99)])) + return { totals, empty } + })) + expect(rows).toEqual({ totals: [{ n: 2, enabled: 1, total: 6, avg: 3, min: 2, max: 4 }], empty: [{ n: 0, sum: null }] }) +}) + +it("round-trips JSON, blobs and zoned timestamps, and runs SQLite scalar functions", async () => { + const Payloads = SQLite.table("payloads", { id: SQLite.integer, value: SQLite.json(Schema.Struct({ message: Schema.String })), bytes: SQLite.blob, at: SQLite.timestamp }) + const seen = await withDatabase((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE payloads (id INTEGER, value TEXT, bytes BLOB, at TEXT)" }) + const returned = yield* db.run(SQLite.insertInto(Payloads).values({ id: 1, value: { message: "__PARAM_id__\0😀" }, bytes: new Uint8Array([0, 127, 255]), at: DateTime.makeUnsafe("2026-10-01T10:00:00+02:00") }).returning()) + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "HELLO", enabled: true, score: null })) + const scalar = yield* db.run(SQLite.from(Items).select(($) => ({ lower: SQLite.lower($.note), upper: SQLite.upper($.note), length: SQLite.length($.note), fallback: SQLite.coalesce($.score, SQLite.lit(9)) }))) + return { returned, scalar } + })) + expect(seen.returned[0]?.value).toEqual({ message: "__PARAM_id__\0😀" }) + expect(seen.returned[0]?.bytes).toEqual(new Uint8Array([0, 127, 255])) + expect(seen.returned.map((row) => DateTime.formatIso(row.at))).toEqual(["2026-10-01T08:00:00.000Z"]) + expect(seen.scalar).toEqual([{ lower: "hello", upper: "HELLO", length: 5, fallback: 9 }]) +}) + +it("binds repeated positional params independently and quotes reserved-marker/NUL literals safely", async () => { + const result = await withDatabase((db) => Effect.gen(function* () { + const note = "' OR 1=1 -- __PARAM_note__\0😀" + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note, enabled: false, score: null })) + const query = SQLite.from(Items).select("id").where(($) => [$.note.eq(SQLite.param.string("note")), $.note.eq(SQLite.param.string("note"))]) + const compiled = yield* SQLite.compile(query, { note }) + const rows = yield* db.run(compiled) + const literal = yield* db.query({ sql: `SELECT ${SQLite.sqliteDialect.quoteString(note)} AS note` }) + return { parameters: compiled.parameters, rows, literal } + })) + expect(result.parameters).toEqual(["' OR 1=1 -- __PARAM_note__\0😀", "' OR 1=1 -- __PARAM_note__\0😀"]) + expect(result.rows).toEqual([{ id: 1 }]) + expect(result.literal).toEqual([{ note: "' OR 1=1 -- __PARAM_note__\0😀" }]) +}) + +it("rejects a stored value that does not match the derived row schema", async () => { + const exit = await withDatabase((db) => Effect.gen(function* () { + yield* db.execute({ sql: "INSERT INTO items VALUES (1, 'bad boolean', 2, NULL)" }) + return yield* Effect.exit(db.run(SQLite.from(Items).select())) + })) + if (!Exit.isFailure(exit)) throw new Error("invalid database row was accepted") + expect(Cause.findError(exit.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) +}) + +it("rolls back failed SQLite transactions and nested savepoints", async () => { + const ids = await withDatabase((db) => Effect.gen(function* () { + const insert = (id: number) => db.run(SQLite.insertInto(Items).values({ id, note: `${id}`, enabled: true, score: null })) + yield* db.transaction(Effect.gen(function* () { + yield* insert(1) + yield* Effect.result(db.transaction(Effect.andThen(insert(2), Effect.fail("inner")))) + yield* insert(3) + })) + yield* Effect.result(db.transaction(Effect.andThen(insert(4), Effect.fail("outer")))) + return yield* db.run(SQLite.from(Items).select("id").orderBy(["id", "asc"])) + })) + expect(ids).toEqual([{ id: 1 }, { id: 3 }]) +}) + +it("executes SQLite upserts, updates and deletes with RETURNING", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([{ id: 1, note: "first", enabled: true, score: 2 }, { id: 2, note: "second", enabled: false, score: null }])) + const upsert = yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "replacement", enabled: true, score: 3 }) + .onConflictDoUpdate({ target: ["id"], set: { note: "updated" } }).returning("id", "note")) + const update = yield* db.run(SQLite.update(Items).set({ note: "changed" }).where(($) => [$.id.eq(2)]).returning("note")) + const deleted = yield* db.run(SQLite.deleteFrom(Items).where(($) => [$.id.eq(2)]).returning("id")) + return { upsert, update, deleted } + })) + expect(rows).toEqual({ upsert: [{ id: 1, note: "updated" }], update: [{ note: "changed" }], deleted: [{ id: 2 }] }) +}) + +it("runs scoped CTEs, joins and derived-table queries with the same row decoder", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "one", enabled: true, score: null })) + const filtered = SQLite.from(Items).select("id", "note").where(($) => [$.id.eq(SQLite.param.int("id"))]) + const Recent = SQLite.table("recent", { id: SQLite.integer, note: SQLite.text }) + const cte = yield* db.run(SQLite.from(Recent).withCTE("recent", filtered).select(), { id: 1 }) + const joined = yield* db.run(SQLite.fromQuery(filtered, "filtered").innerJoin(Items, "i", (a, b) => a.id.eq(b.id)).select(($) => ({ id: $.id, note: $.i.note })), { id: 1 }) + return { cte, joined } + })) + expect(rows).toEqual({ cte: [{ id: 1, note: "one" }], joined: [{ id: 1, note: "one" }] }) +}) + +it("builds an external table from its columns, not from the definition's keys", () => { + const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer, org_id: SQLite.text }, tenantColumn: "org_id" }) + expect(Object.keys(External.columns)).toEqual(["id", "org_id"]) + expect(External.tenantColumn).toBe("org_id") + expect(SQLite.compileUnsafe(SQLite.from(External).select("id").where(($) => [$.org_id.eq("o")])).sql).toContain('"ext"."org_id" = ?') +}) diff --git a/src/sqlite/types.ts b/src/sqlite/types.ts new file mode 100644 index 0000000..a28d9c5 --- /dev/null +++ b/src/sqlite/types.ts @@ -0,0 +1,36 @@ +import { Schema, SchemaGetter } from "effect" +import { brand, custom, type CHType } from "../ch/types" +import { PgTimestampLiteral } from "../pg/types" + +/** A SQLite storage type and the codec for its wire values. */ +export type SqliteType = CHType + +export const text = custom("text", Schema.String) +/** A JavaScript safe integer; use a custom codec for a driver's bigint mode. */ +export const integer = custom("integer", Schema.Int) +export const real = custom("real", Schema.Finite) +/** SQLite stores booleans as 0/1. Other integers are not booleans. */ +export const boolean = custom("integer", Schema.BooleanFromBit) +/** An instant stored as ISO text. */ +export const timestamp = custom("text", Schema.DateTimeUtcFromString, PgTimestampLiteral) + +const bytes = Schema.Array(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 255 }))).pipe( + Schema.decodeTo(Schema.Uint8Array, { + decode: SchemaGetter.transform((value) => new Uint8Array(value)), + encode: SchemaGetter.transform((value) => Array.from(value)), + }), +) +/** Node sends Uint8Array; D1 can send an array of bytes. */ +export const blob = custom("blob", Schema.Union([Schema.Uint8Array, bytes])) +/** JSON is stored as TEXT and decoded through the caller's schema. */ +export const json = (schema: Schema.Codec) => custom("text", Schema.fromJsonString(schema)) + +export const nullable = (type: SqliteType): SqliteType<"Nullable", A | null, I | null> => ({ + _tag: "Nullable", + sql: type.sql, + schema: Schema.NullOr(type.schema), + literalSchema: Schema.NullOr(type.literalSchema), + element: type, +}) + +export { brand, custom } diff --git a/tests/package-consumer-declarations.mts b/tests/package-consumer-declarations.mts index 616c47b..c0561d9 100644 --- a/tests/package-consumer-declarations.mts +++ b/tests/package-consumer-declarations.mts @@ -47,3 +47,13 @@ export const Recent = CH.table("recent", { external: true, columns: { name: CH.s export const withRecent = CH.from(Recent).withCTE("recent", eventsByApp).select(($) => ({ name: $.name })) export const fromSub = CH.fromQuery(eventsByApp, "sub").select(($) => ({ name: $.name })) export const both = CH.unionAll(eventsByApp, CH.from(Events).select(($) => ({ name: $.name }))) + +import * as SQLite from "@maple-dev/effect-orm/sqlite" +export const SqliteItems = SQLite.table("sqlite_items", { id: SQLite.integer, label: SQLite.text }) +export const sqliteById = SQLite.from(SqliteItems).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +export const sqliteCompile = () => SQLite.compile(sqliteById, { id: 1 }) +export const SqliteLinks = SQLite.table("sqlite_links", { + columns: { id: SQLite.text, hits: SQLite.column(SQLite.integer, { default: 0 }) }, + primaryKey: ["id"], + indexes: [SQLite.uniqueIndex("sqlite_links_hits", ["hits"])], +}) diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index 5360120..1b68663 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -91,3 +91,26 @@ assert.equal(typeof runCli, "function") assert.equal(typeof Db.fromSqlClient, "function") assert.equal(Db.isContention(new Db.DatabaseError({ message: "x", sql: "", reason: "SerializationError", cause: undefined })), true) + +// SQLite works from the tarball without installing any SQLite driver. +const SQLite = await import("@maple-dev/effect-orm/sqlite") +const sqliteRows = SQLite.table("consumer_rows", { id: SQLite.integer, note: SQLite.text }) +const sqliteQuery = SQLite.from(sqliteRows).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +const sqliteCompiled = await Effect.runPromise(SQLite.compile(sqliteQuery, { id: 1 })) +assert.deepEqual(sqliteCompiled.parameters, [1]) +assert.equal(sqliteCompiled.dialect, "sqlite") +const sqliteDecoded: ReadonlyArray<{ readonly id: number; readonly note: string }> = await Effect.runPromise(sqliteCompiled.decodeRows([{ id: 1, note: "sqlite" }])) +assert.deepEqual(sqliteDecoded, [{ id: 1, note: "sqlite" }]) +assert.equal(SQLite.compileUnsafe(sqliteQuery, { id: 1 }, { dialect: SQLite.d1Dialect }).dialect, "d1") +const sqliteLinks = SQLite.table("consumer_links", { + columns: { id: SQLite.text, orgId: SQLite.column(SQLite.nullable(SQLite.text), { name: "org_id" }) }, + primaryKey: ["id"], + indexes: [SQLite.index("consumer_links_org", ["orgId"])], +}) +assert.deepEqual(await Effect.runPromise(Effect.map(S.sqliteEntitiesOf([sqliteLinks]), S.renderSqliteSchema)), [ + 'CREATE TABLE IF NOT EXISTS "consumer_links" (\n\t"id" text PRIMARY KEY NOT NULL,\n\t"org_id" text\n)', + 'CREATE INDEX IF NOT EXISTS "consumer_links_org" ON "consumer_links" ("org_id")', +]) +const sqliteNext = await Effect.runPromise(S.sqliteEntitiesOf([sqliteLinks])) +assert.deepEqual(S.diffSqliteSchemas(sqliteNext, sqliteNext).ops, []) +assert.equal((await Effect.runPromise(S.makeSnapshot(sqliteNext, [S.ORIGIN_ID], "sqlite"))).dialect, "sqlite") diff --git a/tsdown.config.ts b/tsdown.config.ts index 7e1f75c..0815e4c 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ expr: "./src/expr.ts", sql: "./src/sql/index.ts", postgres: "./src/postgres.ts", + sqlite: "./src/sqlite.ts", schema: "./src/schema.ts", tinybird: "./src/tinybird.ts", migrate: "./src/migrate.ts",