From 1969a67d315fa35c7c65e9ec0a18a95f95af6f66 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:49:09 +0300 Subject: [PATCH 01/24] feat: add SQLite and D1 codecs and dialect profiles --- bun.lock | 119 +++++++++++++++++++++++++++++++++++++- package.json | 3 + src/sqlite.ts | 28 +++++++++ src/sqlite/dialect.ts | 63 ++++++++++++++++++++ src/sqlite/sqlite.test.ts | 27 +++++++++ src/sqlite/types.ts | 36 ++++++++++++ 6 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 src/sqlite.ts create mode 100644 src/sqlite/dialect.ts create mode 100644 src/sqlite/sqlite.test.ts create mode 100644 src/sqlite/types.ts diff --git a/bun.lock b/bun.lock index db92323..a8a9d5a 100644 --- a/bun.lock +++ b/bun.lock @@ -10,13 +10,16 @@ "@effect/language-service": "^0.87.3", "@effect/platform-bun": "4.0.0", "@effect/sql-clickhouse": "4.0.0", + "@effect/sql-d1": "4.0.0", "@effect/sql-pglite": "4.0.0", + "@effect/sql-sqlite-node": "4.0.0", "@effect/vitest": "4.0.0", "@electric-sql/pglite": "0.5.8", "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", "fast-check": "^4.9.0", + "miniflare": "4.20260730.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9", @@ -68,6 +71,20 @@ "@clickhouse/client-common": ["@clickhouse/client-common@1.21.0", "", {}, "sha512-UjvA32aaLKbQ5U735qAGdzohrOrfU8zFoOWIOWorWQk+7UzHZxq6pf4Uh/wQ08zAMb9cmkd9h9N1FV8zP8UCOg=="], + "@cloudflare/workerd-darwin-64": ["@cloudflare/workerd-darwin-64@1.20260730.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA=="], + + "@cloudflare/workerd-darwin-arm64": ["@cloudflare/workerd-darwin-arm64@1.20260730.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ=="], + + "@cloudflare/workerd-linux-64": ["@cloudflare/workerd-linux-64@1.20260730.1", "", { "os": "linux", "cpu": "x64" }, "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA=="], + + "@cloudflare/workerd-linux-arm64": ["@cloudflare/workerd-linux-arm64@1.20260730.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ=="], + + "@cloudflare/workerd-windows-64": ["@cloudflare/workerd-windows-64@1.20260730.1", "", { "os": "win32", "cpu": "x64" }, "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ=="], + + "@cloudflare/workers-types": ["@cloudflare/workers-types@5.20261024.1", "", {}, "sha512-sNhbgBsuVvmumSurgCcIrXShqACPfRt/XlYFstBeANu5BHEJhXW+e7PmHxDNYRXxKYZV57toUfkLwFp8/0SZKA=="], + + "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@effect/language-service": ["@effect/language-service@0.87.3", "", { "bin": { "effect-language-service": "cli.js" } }, "sha512-g1OmRBZF5chbW9QMNoirPo5Bb4zzU2HfP88tR7/VSZEUxHJ1/qVb8AQ0kUh2ggctLH1I715yCwP4gkw56PTjpQ=="], "@effect/platform-bun": ["@effect/platform-bun@4.0.0", "", { "dependencies": { "@effect/platform-node-shared": "^4.0.0" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-g9O41Sub/+JKfLjvBNEoqR1Pg6wtc2bbYzv7Kw8YsMihbt/ypCn7ZoydEXaHvTTUvbVrXOQbVFlfplyo84Clug=="], @@ -78,14 +95,78 @@ "@effect/sql-clickhouse": ["@effect/sql-clickhouse@4.0.0", "", { "dependencies": { "@clickhouse/client": "^1.23.1" }, "peerDependencies": { "@effect/platform-node": "^4.0.0", "effect": "^4.0.0" } }, "sha512-KthAxSJAW4VR9YyFsgOwwWX2ym6aYMFEJkzyxJExzcGFiUbFMK7hsbwzagiBKJAr/N1gGWQYiZP2fZgkgruiuA=="], + "@effect/sql-d1": ["@effect/sql-d1@4.0.0", "", { "dependencies": { "@cloudflare/workers-types": "^5.20260926.1" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-pmkzncZCGD6CLyvc89keEBU85h+gHVCu/LKQnThd+ruYSUbG0R9jYA+lnk3R8Djp0wQW6SbsbmHQuhopIYyo3Q=="], + "@effect/sql-pglite": ["@effect/sql-pglite@4.0.0", "", { "dependencies": { "@electric-sql/pglite": "^0.5.8" }, "peerDependencies": { "effect": "^4.0.0" } }, "sha512-BTlCPSipKMHZJEoJ94rR4f8Gx8cv8tLpZeHc3sajg2ko/2GY8s6CyycSra3Z6GLFUhcivRTYQ9Zc5LThz64qfQ=="], + "@effect/sql-sqlite-node": ["@effect/sql-sqlite-node@4.0.0", "", { "peerDependencies": { "effect": "^4.0.0" } }, "sha512-xyUSLY7v55iMgGxrjFrTo5ngixgQ3jLmOQ+AX8CCazO/caxuMM0oq8zb974FAcvQHRrpMEWU7Vnoo+rxct7PAQ=="], + "@effect/vitest": ["@effect/vitest@4.0.0", "", { "peerDependencies": { "effect": "^4.0.0", "vitest": ">=5.0.0 <6.0.0" } }, "sha512-kGElCPFGtP+CgMCwV/HjJWqtE3zYo6I1+8MSA54l+EI24lPU9p2Efa+TBp1jfkg9j711KKzEqgSAy3VzTsb76Q=="], "@electric-sql/pglite": ["@electric-sql/pglite@0.5.8", "", {}, "sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA=="], + + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.1" }, "os": "darwin", "cpu": "arm64" }, "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.1" }, "os": "darwin", "cpu": "x64" }, "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw=="], + + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "os": "freebsd" }, "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.1", "", { "os": "linux", "cpu": "none" }, "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.1" }, "os": "linux", "cpu": "arm" }, "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.1" }, "os": "linux", "cpu": "ppc64" }, "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.1" }, "os": "linux", "cpu": "none" }, "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.1" }, "os": "linux", "cpu": "s390x" }, "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" }, "os": "linux", "cpu": "arm64" }, "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.2", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.1" }, "os": "linux", "cpu": "x64" }, "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.2", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw=="], + + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.2", "", { "dependencies": { "@img/sharp-wasm32": "0.35.2" }, "cpu": "none" }, "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.2", "", { "os": "win32", "cpu": "x64" }, "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="], + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], + "@manypkg/find-root": ["@manypkg/find-root@3.1.0", "", { "dependencies": { "@manypkg/tools": "^2.1.0" } }, "sha512-BcSqCyKhBVZ5YkSzOiheMCV41kqAFptW6xGqYSTjkVTl9XQpr+pqHhwgGCOHQtjDCv7Is6EFyA14Sm5GVbVABA=="], "@manypkg/get-packages": ["@manypkg/get-packages@3.1.0", "", { "dependencies": { "@manypkg/find-root": "^3.1.0", "@manypkg/tools": "^2.1.0" } }, "sha512-0TbBVyvPrP7xGYBI/cP8UP+yl/z+HtbTttAD7FMAJgn/kXOTwh5/60TsqP9ZYY710forNfyV0N8P/IE/ujGZJg=="], @@ -96,6 +177,12 @@ "@pnpm/deps.graph-sequencer": ["@pnpm/deps.graph-sequencer@1100.0.1", "", {}, "sha512-pOr5+q1fLYKwFN3LAJuGZEnfXDcQ73zqgDHMtGy+K+uIoUqyY+6MeDCWFwfu+4EFuq76I5EPFofoNAI+Bmmq4A=="], + "@poppinss/colors": ["@poppinss/colors@4.1.6", "", { "dependencies": { "kleur": "^4.1.5" } }, "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg=="], + + "@poppinss/dumper": ["@poppinss/dumper@0.6.5", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@sindresorhus/is": "^7.0.2", "supports-color": "^10.0.0" } }, "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw=="], + + "@poppinss/exception": ["@poppinss/exception@1.2.3", "", {}, "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw=="], + "@quansync/fs": ["@quansync/fs@1.1.0", "", { "dependencies": { "quansync": "^1.0.0" } }, "sha512-qAPG/t3HqML1TlN7sY/pTbEjzFVAKsMjNNMGheyDosro+kT4iw2KCUoHcVdmliwWjorm4elZbgNQyU2eD97eDg=="], "@redis/bloom": ["@redis/bloom@6.3.0", "", { "peerDependencies": { "@redis/client": "^6.3.0" } }, "sha512-NQ5poYpZr0jv6zazN1cy6JY5kIipKFZ3R9GTL4kOUTX3a9YAfY29U8bsE4lWhVQEM+G4V0BXVC57mubIdDijeA=="], @@ -140,6 +227,10 @@ "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "@sindresorhus/is": ["@sindresorhus/is@7.2.0", "", {}, "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw=="], + + "@speed-highlight/core": ["@speed-highlight/core@1.2.24", "", {}, "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], @@ -266,6 +357,8 @@ "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], + "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], + "defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -276,6 +369,8 @@ "empathic": ["empathic@2.1.0", "", {}, "sha512-AnfC1ATldl49/cvZdLPDjBfrRNwbDO05aibiOtzQu3qtlbJtomNLhF30HEtn/7iBz50dlMECqATo3fG0LrdEgw=="], + "error-stack-parser-es": ["error-stack-parser-es@1.0.5", "", {}, "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], @@ -308,6 +403,8 @@ "jsonc-parser": ["jsonc-parser@3.3.1", "", {}, "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ=="], + "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], + "launch-editor": ["launch-editor@2.14.2", "", { "dependencies": { "picocolors": "^1.1.1", "shell-quote": "^1.10.0" } }, "sha512-NHJEb30MGBCkmZXLZwz25t37QR4aY03ZzPYUJXOJaOa92uWwExI9SBQUuIznBYc1xkD7cJNGX7oc3Nl3sq2kQQ=="], "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], @@ -336,6 +433,8 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "miniflare": ["miniflare@4.20260730.0", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", "undici": "7.28.0", "workerd": "1.20260730.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-1Z9SB9r/o//80UA02Re3QhtcecSHAyAjf5EcKBfQVlQrCg7Miy79hl2PvtkwFLIaJ5rcrOPdDcRr577okwZPsg=="], + "nanoid": ["nanoid@3.3.19", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug=="], "obug": ["obug@2.2.1", "", {}, "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q=="], @@ -364,6 +463,8 @@ "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "sharp": ["sharp@0.35.2", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.4" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.2", "@img/sharp-darwin-x64": "0.35.2", "@img/sharp-freebsd-wasm32": "0.35.2", "@img/sharp-libvips-darwin-arm64": "1.3.1", "@img/sharp-libvips-darwin-x64": "1.3.1", "@img/sharp-libvips-linux-arm": "1.3.1", "@img/sharp-libvips-linux-arm64": "1.3.1", "@img/sharp-libvips-linux-ppc64": "1.3.1", "@img/sharp-libvips-linux-riscv64": "1.3.1", "@img/sharp-libvips-linux-s390x": "1.3.1", "@img/sharp-libvips-linux-x64": "1.3.1", "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", "@img/sharp-libvips-linuxmusl-x64": "1.3.1", "@img/sharp-linux-arm": "0.35.2", "@img/sharp-linux-arm64": "0.35.2", "@img/sharp-linux-ppc64": "0.35.2", "@img/sharp-linux-riscv64": "0.35.2", "@img/sharp-linux-s390x": "0.35.2", "@img/sharp-linux-x64": "0.35.2", "@img/sharp-linuxmusl-arm64": "0.35.2", "@img/sharp-linuxmusl-x64": "0.35.2", "@img/sharp-webcontainers-wasm32": "0.35.2", "@img/sharp-win32-arm64": "0.35.2", "@img/sharp-win32-ia32": "0.35.2", "@img/sharp-win32-x64": "0.35.2" } }, "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w=="], + "shell-quote": ["shell-quote@1.12.0", "", {}, "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q=="], "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], @@ -376,6 +477,8 @@ "std-env": ["std-env@4.3.0", "", {}, "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag=="], + "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], "tinyexec": ["tinyexec@1.3.1", "", {}, "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA=="], @@ -388,11 +491,13 @@ "tsdown": ["tsdown@0.22.14", "", { "dependencies": { "ansis": "^4.3.1", "cac": "^7.0.0", "defu": "^6.1.7", "empathic": "^2.0.1", "hookable": "^6.1.1", "import-without-cache": "^0.4.0", "obug": "^2.1.4", "picomatch": "^4.0.5", "rolldown": "~1.2.0", "rolldown-plugin-dts": "^0.27.13", "tinyexec": "^1.2.4", "tinyglobby": "^0.2.17", "tree-kill": "^1.2.2", "unconfig-core": "^7.5.0", "verkit": "^0.3.0" }, "peerDependencies": { "@arethetypeswrong/core": "^0.18.1", "@tsdown/css": "0.22.14", "@tsdown/exe": "0.22.14", "@vitejs/devtools": "*", "publint": "^0.3.8", "tsx": "*", "typescript": "^5.0.0 || ^6.0.0 || ^7.0.0", "unplugin-unused": "^0.5.0", "unrun": "*" }, "optionalPeers": ["@arethetypeswrong/core", "@tsdown/css", "@tsdown/exe", "@vitejs/devtools", "publint", "tsx", "typescript", "unplugin-unused", "unrun"], "bin": { "tsdown": "./dist/run.mjs" } }, "sha512-ule7Y+fsAN2iZbLDoo7C4KYljFJNJJ+fLshyn+9gozeTspVersWHxwdGB+Dm2hzA38s6muFnUTl0jK3vJm9ifQ=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "unconfig-core": ["unconfig-core@7.5.0", "", { "dependencies": { "@quansync/fs": "^1.0.0", "quansync": "^1.0.0" } }, "sha512-Su3FauozOGP44ZmKdHy2oE6LPjk51M/TRRjHv2HNCWiDvfvCoxC2lno6jevMA91MYAdCdwP05QnWdWpSbncX/w=="], - "undici": ["undici@8.11.2", "", {}, "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ=="], + "undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -404,16 +509,26 @@ "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - "ws": ["ws@8.22.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg=="], + "workerd": ["workerd@1.20260730.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260730.1", "@cloudflare/workerd-darwin-arm64": "1.20260730.1", "@cloudflare/workerd-linux-64": "1.20260730.1", "@cloudflare/workerd-linux-arm64": "1.20260730.1", "@cloudflare/workerd-windows-64": "1.20260730.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA=="], + + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], + "youch": ["youch@4.1.0-beta.10", "", { "dependencies": { "@poppinss/colors": "^4.1.5", "@poppinss/dumper": "^0.6.4", "@speed-highlight/core": "^1.2.7", "cookie": "^1.0.2", "youch-core": "^0.3.3" } }, "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ=="], + + "youch-core": ["youch-core@0.3.3", "", { "dependencies": { "@poppinss/exception": "^1.2.2", "error-stack-parser-es": "^1.0.5" } }, "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA=="], + "yuku-ast": ["yuku-ast@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7" } }, "sha512-h6+4bDfyootiMB9vckk5uKo5r5j0GHrkr17FQTDNfEsFT3DWlN9uu1HJwQwc64pgmLCI945fWM3lbTIqxjT3GQ=="], "yuku-codegen": ["yuku-codegen@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7" }, "optionalDependencies": { "@yuku-codegen/binding-android-arm64": "0.8.7", "@yuku-codegen/binding-darwin-arm64": "0.8.7", "@yuku-codegen/binding-darwin-x64": "0.8.7", "@yuku-codegen/binding-freebsd-x64": "0.8.7", "@yuku-codegen/binding-linux-arm-gnu": "0.8.7", "@yuku-codegen/binding-linux-arm-musl": "0.8.7", "@yuku-codegen/binding-linux-arm64-gnu": "0.8.7", "@yuku-codegen/binding-linux-arm64-musl": "0.8.7", "@yuku-codegen/binding-linux-x64-gnu": "0.8.7", "@yuku-codegen/binding-linux-x64-musl": "0.8.7", "@yuku-codegen/binding-win32-arm64": "0.8.7", "@yuku-codegen/binding-win32-x64": "0.8.7" } }, "sha512-adwDZSh8oVDzhE6Du9PwVWxcOxeV0e2EVhUuMKWfhSY4wkrDq9eqixlxFF3l/XGUV1E7UFzhpz9393MUumkyNw=="], "yuku-parser": ["yuku-parser@0.8.7", "", { "dependencies": { "@yuku-toolchain/types": "^0.8.7", "yuku-ast": "^0.8.7" }, "optionalDependencies": { "@yuku-parser/binding-android-arm64": "0.8.7", "@yuku-parser/binding-darwin-arm64": "0.8.7", "@yuku-parser/binding-darwin-x64": "0.8.7", "@yuku-parser/binding-freebsd-x64": "0.8.7", "@yuku-parser/binding-linux-arm-gnu": "0.8.7", "@yuku-parser/binding-linux-arm-musl": "0.8.7", "@yuku-parser/binding-linux-arm64-gnu": "0.8.7", "@yuku-parser/binding-linux-arm64-musl": "0.8.7", "@yuku-parser/binding-linux-x64-gnu": "0.8.7", "@yuku-parser/binding-linux-x64-musl": "0.8.7", "@yuku-parser/binding-win32-arm64": "0.8.7", "@yuku-parser/binding-win32-x64": "0.8.7" } }, "sha512-vRD9nwt4L3aYpxNqeSC4WqLv58xrXef0Ong1Mc45CTXTIpvLafx7JO05sczmQZwdLEZvywrLOGdNC5+Rp5N1BQ=="], + "@effect/platform-node/undici": ["undici@8.11.2", "", {}, "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ=="], + + "@effect/platform-node-shared/ws": ["ws@8.22.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg=="], + "@effect/sql-clickhouse/@clickhouse/client": ["@clickhouse/client@1.23.1", "", {}, "sha512-vs3/Zc1dHvT171btW5nMoPsPCJ6QVJ5pp7obxzO5sjqwFx/jjz9wwCAqcFOdc2DhprugDBaVn+4dVY8hG3A9nw=="], } } diff --git a/package.json b/package.json index 079ebcd..7cbc980 100644 --- a/package.json +++ b/package.json @@ -121,13 +121,16 @@ "@effect/language-service": "^0.87.3", "@effect/platform-bun": "4.0.0", "@effect/sql-clickhouse": "4.0.0", + "@effect/sql-d1": "4.0.0", "@effect/sql-pglite": "4.0.0", + "@effect/sql-sqlite-node": "4.0.0", "@effect/vitest": "4.0.0", "@electric-sql/pglite": "0.5.8", "@types/node": "^22.10.2", "effect": "4.0.0", "expect-type": "^1.3.0", "fast-check": "^4.9.0", + "miniflare": "4.20260730.0", "tsdown": "^0.22.14", "typescript": "^7.0.2", "vitest": "^4.1.9" diff --git a/src/sqlite.ts b/src/sqlite.ts new file mode 100644 index 0000000..25b5ce1 --- /dev/null +++ b/src/sqlite.ts @@ -0,0 +1,28 @@ +// SQLite queries and codecs. DDL/migrations stay with the caller for now. +import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" +import { compiled, compiledUnsafe } from "./ch/failure" +import { sqliteDialect } from "./sqlite/dialect" + +export * from "./core" +export * from "./sqlite/types" +export { table, type TableOptions } from "./ch/table" +export { sqliteDialect, d1Dialect } from "./sqlite/dialect" + +/** The shared compiler, defaulting to SQLite. Pass d1Dialect for D1's limits. */ +export const compile: typeof compileCH = ( + query: Parameters[0], + params?: Record, + options?: Parameters[2], +) => compiled(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) + +/** Compile immediately, throwing on a failed compile. */ +export const compileUnsafe: typeof compileCHUnsafe = ( + query: Parameters[0], + params?: Record, + options?: Parameters[2], +) => compiledUnsafe(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) + +export const compileUnion: typeof compileUnionCH = (query, params, options) => + compileUnionCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) +export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (query, params, options) => + compileUnionUnsafeCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts new file mode 100644 index 0000000..f3961b1 --- /dev/null +++ b/src/sqlite/dialect.ts @@ -0,0 +1,63 @@ +import { DateTime } from "effect" +import { noTransactions, type Dialect } from "../ch/dialect" +import { QueryBuilderError } from "../ch/errors" +import { fail } from "../ch/failure" +import { PARAM_MARKER_PREFIX } from "../ch/param" +import { PgTimestampLiteral, timestampLiteral } from "../pg/types" + +const hex = (value: Uint8Array): string => + Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("") + +/** Hex text keeps NUL and the compiler's reserved param marker out of SQL text. */ +const quoteString = (value: string): string => + value.includes(PARAM_MARKER_PREFIX) || value.includes("\0") + ? `CAST(X'${hex(new TextEncoder().encode(value))}' AS TEXT)` + : `'${value.replace(/'/g, "''")}'` + +const literal = (value: unknown, context: string): string => { + if (value === null) return "NULL" + if (typeof value === "string") return quoteString(value) + if (typeof value === "boolean") return value ? "1" : "0" + if (typeof value === "number" && Number.isFinite(value)) return String(value) + if (value instanceof Uint8Array) return `X'${hex(value)}'` + return fail(new QueryBuilderError({ code: "InvalidLiteral", message: `${context}: cannot write this value as a SQLite literal` }), "NULL") +} + +/** SQLite with positional bindings and transactions supplied by the caller's SqlClient. */ +export const sqliteDialect: Dialect = { + name: "sqlite", + functions: "sqlite", + quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, + quoteString, + literal, + dateTimeLiteral: (value) => quoteString(DateTime.formatIso(value)), + params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766 }, + clauses: { + format: false, + derivedTableAlias: true, + groupByAlias: false, + parenthesizeUnionBranches: false, + returning: true, + onConflict: true, + }, + paramCodecs: { + dateTime: PgTimestampLiteral, + dateTimeSeconds: timestampLiteral((millis) => new Date(Math.floor(millis / 1000) * 1000).toISOString()), + }, + transactions: { + support: "full", + savepoints: true, + isolationLevels: ["serializable"], + accessModes: false, + deferrable: false, + setTransaction: () => undefined, + }, +} + +/** D1 executes SQLite SQL, limits each statement to 100 bindings, and has no interactive transactions. */ +export const d1Dialect: Dialect = { + ...sqliteDialect, + name: "d1", + params: { ...sqliteDialect.params, _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 100 }, + transactions: noTransactions, +} diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts new file mode 100644 index 0000000..a1294f7 --- /dev/null +++ b/src/sqlite/sqlite.test.ts @@ -0,0 +1,27 @@ +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Items = SQLite.table("items", { + id: SQLite.integer, + note: SQLite.text, + enabled: SQLite.boolean, + score: SQLite.nullable(SQLite.real), +}) +const withDatabase = (body: (db: Db.DatabaseApi) => Effect.Effect) => + Effect.runPromise(Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + yield* db.execute({ sql: "CREATE TABLE items (id INTEGER PRIMARY KEY, note TEXT NOT NULL, enabled INTEGER NOT NULL, score REAL DEFAULT 7)" }) + return yield* body(db) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" })))) + +it("runs and decodes SQLite writes and selects through an existing Effect SqlClient", async () => { + const seen = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "O'Reilly\\notes", enabled: true, score: null })) + return yield* db.run(SQLite.from(Items).select().where(($) => [$.enabled.eq(SQLite.param.bool("enabled"))]), { enabled: true }) + })) + expect(seen).toEqual([{ id: 1, note: "O'Reilly\\notes", enabled: true, score: null }]) +}) diff --git a/src/sqlite/types.ts b/src/sqlite/types.ts new file mode 100644 index 0000000..ea6fd18 --- /dev/null +++ b/src/sqlite/types.ts @@ -0,0 +1,36 @@ +import { Schema, SchemaGetter } from "effect" +import { brand, custom, type CHType } from "../ch/types" +import { PgTimestampLiteral } from "../pg/types" + +/** A SQLite storage type and the codec for its wire values. */ +export type SqliteType = CHType + +export const text = custom("text", Schema.String) +/** A JavaScript safe integer; use a custom codec for a driver's bigint mode. */ +export const integer = custom("integer", Schema.Int) +export const real = custom("real", Schema.Finite) +/** SQLite stores booleans as 0/1. Other integers are not booleans. */ +export const boolean = custom("integer", Schema.BooleanFromBit) +/** An explicitly zoned instant stored as ISO text. */ +export const timestamp = custom("text", Schema.DateTimeUtcFromString, PgTimestampLiteral) + +const bytes = Schema.Array(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 255 }))).pipe( + Schema.decodeTo(Schema.Uint8Array, { + decode: SchemaGetter.transform((value) => new Uint8Array(value)), + encode: SchemaGetter.transform((value) => Array.from(value)), + }), +) +/** Node sends Uint8Array; D1 can send an array of bytes. */ +export const blob = custom("blob", Schema.Union([Schema.Uint8Array, bytes])) +/** JSON is stored as TEXT and decoded through the caller's schema. */ +export const json = (schema: Schema.Codec) => custom("text", Schema.fromJsonString(schema)) + +export const nullable = (type: SqliteType): SqliteType<"Nullable", A | null, I | null> => ({ + _tag: "Nullable", + sql: type.sql, + schema: Schema.NullOr(type.schema), + literalSchema: Schema.NullOr(type.literalSchema), + element: type, +}) + +export { brand, custom } From ceb1f051680f76b9566f072d28c230dde834cb79 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:49:28 +0300 Subject: [PATCH 02/24] feat: preserve SQLite offset and union branch pagination --- src/ch/compile.ts | 10 ++++++++-- src/ch/dialect.ts | 4 ++++ src/sqlite/dialect.ts | 2 ++ src/sqlite/sqlite.test.ts | 11 +++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..a6743c9 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -967,7 +967,9 @@ function compileInner< orderBy: state.orderByFn === undefined ? orderByClause(state.orderBySpecs).map(raw) : orderByExprs(state.orderByFn($)).map(raw), - limit: rowCount("limit", state.limitValue), + limit: state.limitValue == null && state.offsetValue != null && currentDialect().clauses.offsetRequiresLimit === true + ? raw("-1") + : rowCount("limit", state.limitValue), offset: rowCount("offset", state.offsetValue), format: options?.skipFormat ? undefined : formatClause(state.formatValue), } @@ -1442,7 +1444,9 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => currentDialect().clauses.wrapUnionBranches === true + ? `SELECT * FROM (\n${q.sql}\n) AS ${quoteIdent("__union_branch")}` + : parenthesize ? `(\n${q.sql}\n)` : q.sql).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = @@ -1455,6 +1459,8 @@ function compileUnionInner, Params extends Re } if (state.outerLimitValue != null) { sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` + } else if (state.outerOffsetValue != null && currentDialect().clauses.offsetRequiresLimit === true) { + sql += "\nLIMIT -1" } if (state.outerOffsetValue != null) { sql += `\nOFFSET ${compileSqlFragment(rowCount("offset", state.outerOffsetValue)!)}` diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b6c97..d20d4b7 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -60,6 +60,10 @@ export interface DialectClauses { /** Whether each `UNION ALL` branch is wrapped in parentheses. Postgres needs * it for a branch with its own WITH, ORDER BY or LIMIT. */ readonly parenthesizeUnionBranches: boolean + /** SQLite preserves branch-local ORDER BY/LIMIT by selecting from each branch. */ + readonly wrapUnionBranches?: boolean + /** SQLite requires LIMIT -1 before OFFSET when the caller omitted a limit. */ + readonly offsetRequiresLimit?: boolean /** `RETURNING` after an INSERT. Absent means no: an insert with * `.returning()` fails to compile for the dialect. */ readonly returning?: boolean diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index f3961b1..ce75ea0 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -37,6 +37,8 @@ export const sqliteDialect: Dialect = { derivedTableAlias: true, groupByAlias: false, parenthesizeUnionBranches: false, + wrapUnionBranches: true, + offsetRequiresLimit: true, returning: true, onConflict: true, }, diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index a1294f7..c303c69 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -25,3 +25,14 @@ it("runs and decodes SQLite writes and selects through an existing Effect SqlCli })) expect(seen).toEqual([{ id: 1, note: "O'Reilly\\notes", enabled: true, score: null }]) }) + +it("keeps query and union pagination when OFFSET has no LIMIT and branches have local limits", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([1, 2, 3].map((id) => ({ id, note: `${id}`, enabled: false, score: null })))) + const offset = yield* db.run(SQLite.from(Items).select("id").orderBy(["id", "asc"]).offset(1)) + const branch = (direction: "asc" | "desc") => SQLite.from(Items).select("id").orderBy(["id", direction]).limit(1) + const union = yield* db.run(SQLite.unionAll(branch("asc"), branch("desc")).orderBy(["id", "asc"]).offset(1)) + return { offset, union } + })) + expect(rows).toEqual({ offset: [{ id: 2 }, { id: 3 }], union: [{ id: 3 }] }) +}) From 4fc495d26ab1e396579606cd9d277b1d20ec8f84 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:50:08 +0300 Subject: [PATCH 03/24] feat: reject SQL clauses SQLite cannot execute --- src/ch/compile.ts | 11 +++++++++-- src/ch/dialect.ts | 7 +++++++ src/ch/expr.ts | 6 ++++-- src/sql/sql-syntax.ts | 2 ++ src/sqlite/dialect.ts | 4 ++++ src/sqlite/sqlite.test.ts | 11 +++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index a6743c9..ac944b2 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -936,7 +936,9 @@ function compileInner< const sqlQuery: SqlQuery = { distinct: state.distinct !== undefined, distinctOn: Array.isArray(state.distinct) - ? (state.distinct.length === 0 + ? (currentDialect().clauses.distinctOn === false + ? fail>(new QueryBuilderDefect({ message: `${currentDialect().name} has no DISTINCT ON clause` }), []) + : state.distinct.length === 0 ? fail>(new QueryBuilderDefect({ message: "CHQuery: distinctOn() needs at least one key" }), []) : state.distinct ).map((key: string) => { @@ -1690,6 +1692,9 @@ const onConflictClause = ( let target = "" if (conflict.target !== undefined) { if ("constraint" in conflict.target) { + if (dialect.clauses.onConflictConstraint === false) { + return fail(new QueryBuilderDefect({ message: `${where}: ${dialect.name} has no ON CONSTRAINT conflict target; name its columns instead` }), "") + } if (conflict.targetWhere !== undefined) { return fail(new QueryBuilderDefect({ message: `${where}: targetWhere needs a column target, not a constraint` }), "") } @@ -1784,7 +1789,9 @@ const valueCells = (table: Table, params: Record = { ...params } let next = 0 const cell = (column: string, value: unknown, context: string): string => { - if (value === undefined) return "DEFAULT" + if (value === undefined) return currentDialect().clauses.defaultInValues === false + ? fail(new QueryBuilderDefect({ message: `${where}: ${currentDialect().name} has no per-cell DEFAULT in VALUES; group rows by their present columns` }), "NULL") + : "DEFAULT" if (isExprLike(value)) return compileSqlFragment(value.toFragment()) const wire = encodeValue(table.columns[column]!.literalSchema, value, `${where}: ${context}, column ${column}`) let name = `${VALUE_PARAM}${next++}` diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index d20d4b7..96ebb93 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -70,6 +70,12 @@ export interface DialectClauses { /** `ON CONFLICT ... DO NOTHING / DO UPDATE` after an INSERT. Absent means * no: an insert with `onConflict*` fails to compile for the dialect. */ readonly onConflict?: boolean + /** Whether ON CONFLICT can name a constraint rather than columns. */ + readonly onConflictConstraint?: boolean + /** Whether a VALUES tuple can contain DEFAULT for a missing cell. */ + readonly defaultInValues?: boolean + /** Whether SELECT DISTINCT ON is supported. Undefined preserves existing dialects. */ + readonly distinctOn?: boolean /** `SETTINGS` on an INSERT, UPDATE or DELETE (ClickHouse). Absent means * no: a write with `.settings()` fails to compile for the dialect. */ readonly writeSettings?: boolean @@ -210,6 +216,7 @@ export const checkedLiteral = (dialect: Dialect, value: unknown, context: string const checkedSyntax = (dialect: Dialect): SqlSyntax => ({ dialect: dialect.name, + ilike: dialect.ilike, quoteIdent: dialect.quoteIdent, quoteString: (value) => checked(dialect, dialect.quoteString(value), "a string literal"), literal: (value, context) => checkedLiteral(dialect, value, context), diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 99773df..34bb9f1 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -13,7 +13,7 @@ import { activeSqlSyntax } from "../sql/sql-syntax" import { activeLiteralBinder } from "../sql/literal-binder" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferEncoded, type InferTS } from "./types" import { encodeColumnLiteral, isSubnormal, numberText, sqlLiteral } from "./literal" -import { QueryBuilderError } from "./errors" +import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { fail } from "./failure" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" @@ -533,7 +533,9 @@ export function makeExpr( like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}`)), notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}`)), - ilike: (pattern) => makeCond(known(() => `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), + ilike: (pattern) => makeCond(known(() => activeSqlSyntax()?.ilike === false + ? fail(new QueryBuilderDefect({ message: `ILIKE is not supported by the ${activeSqlSyntax()?.dialect} dialect; use LIKE with SQLite's own case rules` }), "FALSE") + : `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))), notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))), diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index bf0fe19..89106c7 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -11,6 +11,8 @@ import type { DateTime } from "effect" export interface SqlSyntax { /** The dialect's name (`clickhouse`, `postgres`, …), for the few choices expression code makes per dialect. */ readonly dialect: string + /** False when the dialect has no ILIKE operator. */ + readonly ilike?: boolean /** One identifier (a column, alias, table or schema name), quoted as needed. */ readonly quoteIdent: (name: string) => string /** A string as a quoted literal. */ diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index ce75ea0..622c030 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -27,6 +27,7 @@ const literal = (value: unknown, context: string): string => { export const sqliteDialect: Dialect = { name: "sqlite", functions: "sqlite", + ilike: false, quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, quoteString, literal, @@ -41,6 +42,9 @@ export const sqliteDialect: Dialect = { offsetRequiresLimit: true, returning: true, onConflict: true, + onConflictConstraint: false, + defaultInValues: false, + distinctOn: false, }, paramCodecs: { dateTime: PgTimestampLiteral, diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index c303c69..820f864 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -36,3 +36,14 @@ it("keeps query and union pagination when OFFSET has no LIMIT and branches have })) expect(rows).toEqual({ offset: [{ id: 2 }, { id: 3 }], union: [{ id: 3 }] }) }) + +it("refuses unsupported SQLite syntax while compiling, before sending SQL", () => { + const defaulted = SQLite.table("items", { id: SQLite.integer, note: SQLite.text }, { defaults: ["note"] }) + expect(() => SQLite.compileUnsafe(SQLite.insertInto(defaulted).values([{ id: 1, note: "a" }, { id: 2 }]))).toThrow(/DEFAULT/) + expect(() => SQLite.compileUnsafe(SQLite.insertInto(Items) + .values({ id: 1, note: "a", enabled: true, score: null }) + .onConflictDoNothing({ target: { constraint: "items_pkey" } }))).toThrow(/ON CONSTRAINT/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").distinctOn("id"))).toThrow(/DISTINCT ON/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.note.ilike("a%")]))).toThrow(/ILIKE/) + expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").forUpdate())).toThrow(/row locks/) +}) From f74a9db38543f3a89301a8cde716f4b9ff6e87d8 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:51:38 +0300 Subject: [PATCH 04/24] fix: disambiguate SQLite insert-select upserts --- src/ch/compile.ts | 4 +++- src/ch/dialect.ts | 2 ++ src/sqlite/dialect.ts | 1 + src/sqlite/sqlite.test.ts | 9 +++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index ac944b2..cb2a4e5 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -1967,7 +1967,9 @@ function compileInsert(insert: CHInsert, params: Record SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.note.ilike("a%")]))).toThrow(/ILIKE/) expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").forUpdate())).toThrow(/row locks/) }) + +it("runs INSERT SELECT with ON CONFLICT even when the source has no WHERE", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "first", enabled: true, score: null })) + yield* db.run(SQLite.insertInto(Items).select(SQLite.from(Items).select()).onConflictDoNothing()) + return yield* db.run(SQLite.from(Items).select("id")) + })) + expect(rows).toEqual([{ id: 1 }]) +}) From 80cf8a94770310c7ff16b5c6a0e0f1363af6809c Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:52:55 +0300 Subject: [PATCH 05/24] feat: add schema-aware SQLite aggregate and scalar functions --- src/ch/functions/builtin.ts | 3 ++- src/sqlite.ts | 1 + src/sqlite/functions.ts | 37 +++++++++++++++++++++++++++++++++++++ src/sqlite/sqlite.test.ts | 13 +++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/sqlite/functions.ts diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 766d2a5..7e63f68 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -21,10 +21,11 @@ import { QueryBuilderDefect } from "../errors" import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "../expr" import type { FnResult } from "../define-fn" -export type FunctionSet = "clickhouse" | "postgres" +export type FunctionSet = "clickhouse" | "postgres" | "sqlite" export type BuiltinKind = "scalar" | "aggregate" | "window" const setLabel: Record = { + sqlite: "a SQLite function (from @maple-dev/effect-orm/sqlite)", clickhouse: "a ClickHouse function (from @maple-dev/effect-orm/clickhouse)", postgres: "a Postgres function (from @maple-dev/effect-orm/postgres)", } diff --git a/src/sqlite.ts b/src/sqlite.ts index 25b5ce1..4f04a9a 100644 --- a/src/sqlite.ts +++ b/src/sqlite.ts @@ -5,6 +5,7 @@ import { sqliteDialect } from "./sqlite/dialect" export * from "./core" export * from "./sqlite/types" +export * from "./sqlite/functions" export { table, type TableOptions } from "./ch/table" export { sqliteDialect, d1Dialect } from "./sqlite/dialect" diff --git a/src/sqlite/functions.ts b/src/sqlite/functions.ts new file mode 100644 index 0000000..afcc9b6 --- /dev/null +++ b/src/sqlite/functions.ts @@ -0,0 +1,37 @@ +import { Schema } from "effect" +import { makeExpr, type Condition, type Expr } from "../ch/expr" +import { schemaOf } from "../ch/define-fn" +import { builtins } from "../ch/functions/builtin" +import { compile } from "../sql/sql-fragment" + +const aggregate = builtins("sqlite", "aggregate") +const scalar = builtins("sqlite", "scalar") +const sql = (expr: Expr | Condition): string => compile(expr.toFragment()) +const nullableNumber = Schema.NullOr(Schema.Finite) + +export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), Schema.Int) +export const countDistinct =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`, "count"), Schema.Int) +export const countIf =

(condition: Condition

): Expr => + makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`, "count"), Schema.Int) +export const sum =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`, "sum"), nullableNumber) +export const avg =

(expr: Expr): Expr => + makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`, "avg"), nullableNumber) +export const min = (expr: Expr): Expr => { + const schema = schemaOf(expr) + return makeExpr(aggregate.lazy(() => `min(${sql(expr)})`, "min"), schema === undefined ? undefined : Schema.NullOr(schema)) +} +export const max = (expr: Expr): Expr => { + const schema = schemaOf(expr) + return makeExpr(aggregate.lazy(() => `max(${sql(expr)})`, "max"), schema === undefined ? undefined : Schema.NullOr(schema)) +} +export const lower =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `lower(${sql(expr)})`, "lower"), Schema.NullOr(Schema.String)) +export const upper =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `upper(${sql(expr)})`, "upper"), Schema.NullOr(Schema.String)) +/** Text length in SQLite characters, or blob length in bytes. */ +export const length =

(expr: Expr): Expr => + makeExpr(scalar.lazy(() => `length(${sql(expr)})`, "length"), Schema.NullOr(Schema.Int)) +export const coalesce = (first: Expr, fallback: Expr): Expr => + makeExpr(scalar.lazy(() => `coalesce(${sql(first)}, ${sql(fallback)})`, "coalesce"), schemaOf(fallback)) diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index de5ebb5..f1571c8 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -56,3 +56,16 @@ it("runs INSERT SELECT with ON CONFLICT even when the source has no WHERE", asyn })) expect(rows).toEqual([{ id: 1 }]) }) + +it("decodes SQLite aggregates, including SQL NULL for an empty aggregate", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([ + { id: 1, note: "A", enabled: true, score: 2 }, + { id: 2, note: "B", enabled: false, score: 4 }, + ])) + const totals = yield* db.run(SQLite.from(Items).select(($) => ({ n: SQLite.count(), enabled: SQLite.countIf($.enabled.eq(true)), total: SQLite.sum($.score), avg: SQLite.avg($.score), min: SQLite.min($.score), max: SQLite.max($.score) }))) + const empty = yield* db.run(SQLite.from(Items).select(($) => ({ n: SQLite.count(), sum: SQLite.sum($.score) })).where(($) => [$.id.eq(99)])) + return { totals, empty } + })) + expect(rows).toEqual({ totals: [{ n: 2, enabled: 1, total: 6, avg: 3, min: 2, max: 4 }], empty: [{ n: 0, sum: null }] }) +}) From 65e4bfabdac430a6e034bacf373b595cc3a26482 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 14:54:36 +0300 Subject: [PATCH 06/24] test: verify SQLite transactions and local D1 safety checks --- src/sqlite/d1.test.ts | 88 +++++++++++++++++++++++++++++++++++++++ src/sqlite/sqlite.test.ts | 55 +++++++++++++++++++++++- 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/sqlite/d1.test.ts diff --git a/src/sqlite/d1.test.ts b/src/sqlite/d1.test.ts new file mode 100644 index 0000000..ff9c123 --- /dev/null +++ b/src/sqlite/d1.test.ts @@ -0,0 +1,88 @@ +// A real local Workers D1 binding; no cloud account or network database needed. +import * as D1Client from "@effect/sql-d1/D1Client" +import { Cause, Effect, Exit, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { Miniflare } from "miniflare" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Ack = SQLite.table("whats_new_ack", { + id: SQLite.text, + entry_id: SQLite.text, + user_id: SQLite.text, + org_id: SQLite.text, + acked_at: SQLite.text, +}, { tenantColumn: "org_id" }) +const first = { id: "ack_1", entry_id: "release_1", user_id: "user_1", org_id: "org_1", acked_at: "2026-10-01T09:00:00.000Z" } +const second = { ...first, id: "ack_2", entry_id: "release_2" } +const other = { ...first, id: "ack_3", user_id: "user_2", org_id: "org_2" } + +const withD1 = async (body: (db: Db.DatabaseApi, statements: Array) => Effect.Effect) => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('D1 dialect test') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + try { + const binding = await mf.getD1Database("DB") + return await Effect.runPromise(Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + const statements: Array = [] + const db = Db.fromSqlClient(client, { + dialect: SQLite.d1Dialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* db.execute({ sql: "CREATE TABLE whats_new_ack (id TEXT PRIMARY KEY, entry_id TEXT NOT NULL, user_id TEXT NOT NULL, org_id TEXT NOT NULL, acked_at TEXT NOT NULL, UNIQUE(entry_id, user_id))" }) + statements.length = 0 + return yield* body(db, statements) + }).pipe(Effect.provide(D1Client.layer({ db: binding })))) + } finally { + await mf.dispose() + } +} + +it("copies acknowledgments on D1, keeps the first row, scopes reads and pages explicitly across tenants", async () => { + const result = await withD1((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Ack).values([first, second, other]).onConflictDoNothing().crossTenant()) + yield* db.run(SQLite.insertInto(Ack).values({ ...first, id: "ack_9", acked_at: "2026-10-09T00:00:00.000Z" }).onConflictDoNothing()) + const byUser = yield* db.run(SQLite.from(Ack).select("entry_id").where(($) => [$.org_id.eq("org_1"), $.user_id.eq("user_1")]).orderBy(["entry_id", "asc"])) + const page = yield* db.run(SQLite.from(Ack).select().where(($) => [$.id.gt("ack_1")]).orderBy(["id", "asc"]).limit(2).crossTenant()) + const kept = yield* db.run(SQLite.from(Ack).select().where(($) => [$.org_id.eq("org_1"), $.id.eq("ack_1")])) + return { byUser, page, kept } + })) + expect(result).toEqual({ byUser: [{ entry_id: "release_1" }, { entry_id: "release_2" }], page: [second, other], kept: [first] }) +}, 30_000) + +it("refuses an unscoped query, too many bindings and interactive transactions before dispatch", async () => { + const seen = await withD1((db, statements) => Effect.gen(function* () { + const scope = yield* Effect.exit(db.run(SQLite.from(Ack).select())) + const limits = yield* Effect.exit(db.run(SQLite.insertInto(Ack).values(Array.from({ length: 21 }, (_, i) => ({ ...first, id: `${i}` }))).crossTenant())) + let bodyRan = false + const transaction = yield* Effect.exit(db.transaction(Effect.sync(() => { bodyRan = true }))) + return { scope, limits, transaction, bodyRan, statements: [...statements] } + })) + if (!Exit.isFailure(seen.scope) || !Exit.isFailure(seen.limits) || !Exit.isFailure(seen.transaction)) throw new Error("a D1 guard accepted an invalid request") + expect(Cause.findDefect(seen.scope.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/TenantScopeViolation" } }) + expect(Cause.findError(seen.limits.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/QueryBuilderError" } }) + expect(Cause.findError(seen.transaction.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/TransactionUnsupported" } }) + expect(seen.bodyRan).toBe(false) + expect(seen.statements).toEqual([]) +}, 30_000) + +it("decodes D1 JSON and blob wire values and returns typed errors for malformed rows", async () => { + const Payloads = SQLite.table("payloads", { id: SQLite.integer, payload: SQLite.json(Schema.Struct({ ok: Schema.Boolean })), bytes: SQLite.blob }) + const seen = await withD1((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE payloads (id INTEGER, payload TEXT, bytes BLOB)" }) + yield* db.run(SQLite.insertInto(Payloads).values({ id: 1, payload: { ok: true }, bytes: new Uint8Array([0, 127, 255]) })) + const rows = yield* db.run(SQLite.from(Payloads).select()) + yield* db.execute({ sql: "UPDATE payloads SET payload = '{}' WHERE id = 1" }) + const malformed = yield* Effect.exit(db.run(SQLite.from(Payloads).select())) + return { rows, malformed } + })) + expect(seen.rows).toEqual([{ id: 1, payload: { ok: true }, bytes: new Uint8Array([0, 127, 255]) }]) + if (!Exit.isFailure(seen.malformed)) throw new Error("malformed JSON was accepted") + expect(Cause.findError(seen.malformed.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) +}, 30_000) diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index f1571c8..303848e 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -1,5 +1,5 @@ import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" -import { Effect } from "effect" +import { Cause, DateTime, Effect, Exit, Schema } from "effect" import * as SqlClient from "effect/sql/SqlClient" import { expect, it } from "vitest" import * as Db from "../database" @@ -69,3 +69,56 @@ it("decodes SQLite aggregates, including SQL NULL for an empty aggregate", async })) expect(rows).toEqual({ totals: [{ n: 2, enabled: 1, total: 6, avg: 3, min: 2, max: 4 }], empty: [{ n: 0, sum: null }] }) }) + +it("round-trips JSON, blobs and zoned timestamps, and runs SQLite scalar functions", async () => { + const Payloads = SQLite.table("payloads", { id: SQLite.integer, value: SQLite.json(Schema.Struct({ message: Schema.String })), bytes: SQLite.blob, at: SQLite.timestamp }) + const seen = await withDatabase((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE payloads (id INTEGER, value TEXT, bytes BLOB, at TEXT)" }) + const returned = yield* db.run(SQLite.insertInto(Payloads).values({ id: 1, value: { message: "__PARAM_id__\0😀" }, bytes: new Uint8Array([0, 127, 255]), at: DateTime.makeUnsafe("2026-10-01T10:00:00+02:00") }).returning()) + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "HELLO", enabled: true, score: null })) + const scalar = yield* db.run(SQLite.from(Items).select(($) => ({ lower: SQLite.lower($.note), upper: SQLite.upper($.note), length: SQLite.length($.note), fallback: SQLite.coalesce($.score, SQLite.lit(9)) }))) + return { returned, scalar } + })) + expect(seen.returned[0]?.value).toEqual({ message: "__PARAM_id__\0😀" }) + expect(seen.returned[0]?.bytes).toEqual(new Uint8Array([0, 127, 255])) + expect(seen.returned.map((row) => DateTime.formatIso(row.at))).toEqual(["2026-10-01T08:00:00.000Z"]) + expect(seen.scalar).toEqual([{ lower: "hello", upper: "HELLO", length: 5, fallback: 9 }]) +}) + +it("binds repeated positional params independently and quotes reserved-marker/NUL literals safely", async () => { + const result = await withDatabase((db) => Effect.gen(function* () { + const note = "' OR 1=1 -- __PARAM_note__\0😀" + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note, enabled: false, score: null })) + const query = SQLite.from(Items).select("id").where(($) => [$.note.eq(SQLite.param.string("note")), $.note.eq(SQLite.param.string("note"))]) + const compiled = yield* SQLite.compile(query, { note }) + const rows = yield* db.run(compiled) + const literal = yield* db.query({ sql: `SELECT ${SQLite.sqliteDialect.quoteString(note)} AS note` }) + return { parameters: compiled.parameters, rows, literal } + })) + expect(result.parameters).toEqual(["' OR 1=1 -- __PARAM_note__\0😀", "' OR 1=1 -- __PARAM_note__\0😀"]) + expect(result.rows).toEqual([{ id: 1 }]) + expect(result.literal).toEqual([{ note: "' OR 1=1 -- __PARAM_note__\0😀" }]) +}) + +it("rejects a stored value that does not match the derived row schema", async () => { + const exit = await withDatabase((db) => Effect.gen(function* () { + yield* db.execute({ sql: "INSERT INTO items VALUES (1, 'bad boolean', 2, NULL)" }) + return yield* Effect.exit(db.run(SQLite.from(Items).select())) + })) + if (!Exit.isFailure(exit)) throw new Error("invalid database row was accepted") + expect(Cause.findError(exit.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) +}) + +it("rolls back failed SQLite transactions and nested savepoints", async () => { + const ids = await withDatabase((db) => Effect.gen(function* () { + const insert = (id: number) => db.run(SQLite.insertInto(Items).values({ id, note: `${id}`, enabled: true, score: null })) + yield* db.transaction(Effect.gen(function* () { + yield* insert(1) + yield* Effect.result(db.transaction(Effect.andThen(insert(2), Effect.fail("inner")))) + yield* insert(3) + })) + yield* Effect.result(db.transaction(Effect.andThen(insert(4), Effect.fail("outer")))) + return yield* db.run(SQLite.from(Items).select("id").orderBy(["id", "asc"])) + })) + expect(ids).toEqual([{ id: 1 }, { id: 3 }]) +}) From 20b9478c6efcaf14fbc86ada029ee96c1459ee03 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sat, 24 Oct 2026 15:03:35 +0300 Subject: [PATCH 07/24] docs: publish the SQLite entry and document its tested limits --- .changeset/quiet-sqlite-dialects.md | 8 ++ README.md | 5 +- docs/README.md | 4 + docs/reference.md | 27 +++++- docs/sqlite.md | 115 ++++++++++++++++++++++++ package.json | 11 ++- scripts/check-exports-documented.mjs | 4 +- src/sqlite/sqlite.test-d.ts | 18 ++++ src/sqlite/sqlite.test.ts | 24 +++++ src/sqlite/types.ts | 2 +- tests/package-consumer-declarations.mts | 5 ++ tests/package-consumer.mts | 11 +++ tsdown.config.ts | 1 + 13 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 .changeset/quiet-sqlite-dialects.md create mode 100644 docs/sqlite.md create mode 100644 src/sqlite/sqlite.test-d.ts diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md new file mode 100644 index 0000000..c4b380e --- /dev/null +++ b/.changeset/quiet-sqlite-dialects.md @@ -0,0 +1,8 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Add the `/sqlite` query entry with SQLite codecs, functions and positional bindings. Use +`sqliteDialect` with an Effect SQLite SqlClient, or `d1Dialect` for Cloudflare D1's binding limit +and explicit refusal of interactive transactions. SQLite schema/migration generation is not +implemented yet. diff --git a/README.md b/README.md index 4863efe..4ad708a 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ > **Alpha.** The API is still changing and minor releases may break it. Pin an exact version, > read the [changelog](./CHANGELOG.md) before upgrading, and expect rough edges. -Type-safe ClickHouse and Postgres queries, result decoding, and reproducible benchmarks for Effect and TypeScript. +Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproducible benchmarks for Effect and TypeScript. Formerly `@maple-dev/effect-clickhouse`. +SQLite and Cloudflare D1 use the same builder and row decoder through the new +[`/sqlite` entry](./docs/sqlite.md). SQLite schema/migration generation is not implemented. + [Read the documentation](https://effect-clickhouse.maple.dev) · [Getting started](./docs/getting-started.md) · [Recipes](./docs/recipes.md) diff --git a/docs/README.md b/docs/README.md index 7a61d11..cbdb224 100644 --- a/docs/README.md +++ b/docs/README.md @@ -99,3 +99,7 @@ The optional `@maple-dev/effect-orm/benchmark` entry point and bundled `ch-bench` CLI measure real queries, compare fixed workloads, and save evidence. See [Benchmarking](./benchmarking.md) and the [agent playbook](./benchmark-agent.md). The SQL builders remain driver-free. + +## SQLite and D1 + +[SQLite and D1](./sqlite.md) covers `@maple-dev/effect-orm/sqlite`, its codecs and functions, Effect SQL drivers, tenant checks and transaction limits. SQLite migration generation is not implemented. diff --git a/docs/reference.md b/docs/reference.md index abc2d46..d6c67c2 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -5,9 +5,10 @@ The package has one entry per database, each self-contained: ```ts import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" +import * as SQLite from "@maple-dev/effect-orm/sqlite" ``` -Both carry the whole [query builder](#query-builder-both-entries). Each adds its own column +All carry the whole [query builder](#query-builder-both-entries). Each adds its own column types, functions, table definitions and a `compile` that defaults to its dialect: [`/clickhouse`](#clickhouse) and [`/postgres`](#postgres). Everything else is on a [subpath](#other-subpaths) for a narrower job. @@ -34,7 +35,7 @@ uniformly, which some codebases prefer for exactly this reason. ## Query builder (both entries) -Everything in this section is exported, identically, from both `/clickhouse` and `/postgres`. +Everything in this section is exported, identically, from `/clickhouse`, `/postgres` and `/sqlite`. ### Query construction @@ -530,3 +531,25 @@ verification, and JSON protocol. Note `/sql` exports a `compile` (fragment → string) distinct from the dialect entries' `compile` (query → `CompiledQuery`), and a `when` distinct from the query builder's `when` (optional conditions). + +## SQLite and D1 + +`@maple-dev/effect-orm/sqlite` exports the shared query builder and the four compile functions. +`sqliteDialect` is the default; `d1Dialect` uses the same SQL with a 100-binding limit and no +interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and limits. + +| Export | Purpose | +| --- | --- | +| `SqliteType` | A SQLite SQL type with its Effect codec | +| `text`, `integer`, `real` | Text, safe integer and finite number codecs | +| `boolean` | A boolean encoded as 0/1; other integers fail decoding | +| `timestamp` | An instant stored as ISO text | +| `blob` | Bytes from a Uint8Array or D1's array wire representation | +| `json(schema)` | JSON text decoded through the supplied schema | +| `nullable(type)`, `custom`, `brand` | Type composition | +| `table(name, columns, options?)`, `TableOptions` | A query schema, including `tenantColumn`, defaults and computed-column metadata; no generated DDL | +| `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min`, `max` | SQLite aggregates; the last four can return SQL NULL | +| `lower`, `upper`, `length`, `coalesce` | SQLite scalar functions | + +SQLite's table definition is for query typing and decoding. The schema/kit/migrate entries +currently support ClickHouse and Postgres DDL only. diff --git a/docs/sqlite.md b/docs/sqlite.md new file mode 100644 index 0000000..2491ca9 --- /dev/null +++ b/docs/sqlite.md @@ -0,0 +1,115 @@ +# SQLite and Cloudflare D1 + +Import `@maple-dev/effect-orm/sqlite` for the shared query builder, SQLite codecs and functions. +Compilation uses positional `?` bindings. Each occurrence of a parameter gets its own binding. +The package opens no connection: use the Effect `SqlClient` you already provide. + +## Define and compile a query + +```ts title="sqlite-quickstart.ts" +import * as SQLite from "@maple-dev/effect-orm/sqlite" + +export const Acks = SQLite.table("acknowledgments", { + id: SQLite.text, + org_id: SQLite.text, + entry_id: SQLite.text, +}, { tenantColumn: "org_id" }) + +export const byOrg = SQLite.from(Acks) + .select("entry_id") + .where(($) => [$.org_id.eq(SQLite.param.string("orgId"))]) + .orderBy(["entry_id", "asc"]) + +export const compiled = SQLite.compileUnsafe(byOrg, { orgId: "org_1" }) +// compiled.parameters is ["org_1"]; compiled.tenantScope is "single-tenant". +``` + +`table` describes the query schema. Manage its actual DDL and migrations outside this package. +Unlike the ClickHouse and Postgres entries, SQLite has no schema diff, introspection, migration +ledger or kit dialect yet. A `table` value here cannot be passed to the DDL generators. + +The entry exports `text`, `integer` (a safe JavaScript integer), `real` (finite numbers), +`boolean` (0/1), `blob`, `timestamp` (ISO text), `json(schema)`, `nullable`, `custom` and `brand`. +`blob` accepts the Uint8Array wire form from Node and the array-of-bytes form from D1. +`json` always uses the supplied schema to validate decoded JSON. +Use `text` when a stored date/time must remain a string. + +SQLite aggregates include `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min` and `max`. +The last four return NULL over an empty input. Scalar helpers include `lower`, `upper`, +`length` and `coalesce`. SQLite's own text and case rules apply. Foreign built-in function +sets, such as ClickHouse's `count()`, fail compilation instead of producing invalid SQL. + +## Execute on SQLite + +Install a compatible Effect SQL driver, for example `@effect/sql-sqlite-node` on Node 24+. +No driver is a runtime dependency of this package. Use current SQLite with RETURNING support. + +```ts title="sqlite-node.ts" +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import { sqliteDialect } from "@maple-dev/effect-orm/sqlite" + +export const SQLiteLive = Db.layerSqlClient({ + dialect: sqliteDialect, + requireTenantScope: true, +}).pipe(Layer.provide(SqliteClient.layer({ filename: ":memory:" }))) +``` + +`Db.run(query, params)` compiles for the database dialect and decodes the selected rows. +SQLite transactions and nested savepoints use the supplied client's `withTransaction`. +`serializable` is the only accepted explicit isolation level. Per-transaction access modes +and deferrable settings are refused. The driver must support the declared transaction behavior; +use the D1 profile for a D1 client. + +## Execute on D1 + +Install `@effect/sql-d1` with a compatible Effect version. Provide the Worker's D1 binding: + +```ts title="sqlite-d-one.ts" +import * as D1Client from "@effect/sql-d1/D1Client" +import { Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import { d1Dialect } from "@maple-dev/effect-orm/sqlite" + +export const d1DatabaseLayer = (db: D1Client.D1ClientConfig["db"]) => + Db.layerSqlClient({ dialect: d1Dialect, requireTenantScope: true }).pipe( + Layer.provide(D1Client.layer({ db })), + ) +``` + +Use `d1Dialect` when compiling a query separately as well: +`SQLite.compile(query, params, { dialect: SQLite.d1Dialect })`. A query compiled for `sqlite` +cannot later run on a database configured as `d1`; pass the builder to `Db.run` instead. + +D1 compilation refuses more than [100 bound parameters per statement](https://developers.cloudflare.com/d1/platform/limits/). +It never splits a write implicitly. Chunk an import explicitly if needed; several chunks do +not make one atomic write. SQLite compilation uses its modern default limit of 32766; a driver +with a lower limit needs a custom dialect profile. + +`Database.transaction` on D1 returns `TransactionUnsupported` before executing its body. +D1's [atomic batch API](https://developers.cloudflare.com/d1/worker-api/d1-database/#batch) +executes a fixed list of statements. It cannot run an arbitrary Effect transaction body with +reads that determine later writes. There is no ORM batch API in this first implementation. + +## Safety and syntax limits + +Derived schemas validate returned rows. A row with the wrong type returns +`CompiledQueryDecodeError`. Input values use each column's encoder before binding. + +Tenant enforcement is **opt-in**, through `requireTenantScope: true`. A compiled query over a +tenant table must pin its `tenantColumn` to one value, or explicitly use `.crossTenant()` for +an administrative read/write. Raw SQL passed to `query` or `execute` has no tenant proof and +bypasses this guard. It is not a replacement for an application's authorization checks. + +The dialect handles OFFSET without LIMIT, branch-local ordering/limits in UNION ALL, and +INSERT SELECT with ON CONFLICT. It refuses these unsupported forms at compile time: + +- `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE` with SQLite's own case rules. +- `ON CONSTRAINT` conflict targets. Name the indexed columns instead. +- Per-cell `DEFAULT` in a VALUES tuple. Omit a defaulted column from every row in the statement, + or group rows by their present columns. All-default rows are not supported by the builder. +- ClickHouse FORMAT/SETTINGS and built-in functions from another dialect. + +The tests run real SQLite through `@effect/sql-sqlite-node` and local workerd/D1 through +Miniflare. They require no deployed database or Cloudflare credentials. diff --git a/package.json b/package.json index 7cbc980..5238372 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@maple-dev/effect-orm", "version": "0.7.2", - "description": "Type-safe ClickHouse and Postgres queries, result decoding, and reproducible benchmarks for Effect and TypeScript", + "description": "Type-safe ClickHouse, Postgres and SQLite queries, result decoding, and reproducible benchmarks for Effect and TypeScript", "keywords": [ "clickhouse", "effect", @@ -9,7 +9,9 @@ "postgres", "query-builder", "sql", - "typescript" + "typescript", + "sqlite", + "cloudflare-d1" ], "homepage": "https://effect-clickhouse.maple.dev", "bugs": { @@ -97,6 +99,11 @@ "types": "./dist/benchmark/cli.d.mts", "import": "./dist/benchmark/cli.mjs", "default": "./dist/benchmark/cli.mjs" + }, + "./sqlite": { + "types": "./dist/sqlite.d.mts", + "import": "./dist/sqlite.mjs", + "default": "./dist/sqlite.mjs" } }, "publishConfig": { diff --git a/scripts/check-exports-documented.mjs b/scripts/check-exports-documented.mjs index 93bec71..0aa5ff8 100644 --- a/scripts/check-exports-documented.mjs +++ b/scripts/check-exports-documented.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Every export of the two dialect entries is named in docs/reference.md, whose +// Every export of the three dialect entries is named in docs/reference.md, whose // first line claims to be the catalog. A claim like that decays the moment // anything is added, so it is checked rather than promised. // @@ -11,7 +11,7 @@ import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" const root = join(dirname(fileURLToPath(import.meta.url)), "..") -const entries = ["clickhouse", "postgres"] +const entries = ["clickhouse", "postgres", "sqlite"] const exportsOf = (entry) => { const dts = join(root, `dist/${entry}.d.mts`) diff --git a/src/sqlite/sqlite.test-d.ts b/src/sqlite/sqlite.test-d.ts new file mode 100644 index 0000000..aa2f74d --- /dev/null +++ b/src/sqlite/sqlite.test-d.ts @@ -0,0 +1,18 @@ +import type { Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as SQLite from "../sqlite" + +const Items = SQLite.table("items", { id: SQLite.integer, note: SQLite.text }) +const query = SQLite.from(Items).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +const compiled = SQLite.compileUnsafe(query, { id: 1 }) +expectTypeOf>>().toEqualTypeOf>() +const effect = SQLite.compile(query, { id: 1 }) +expectTypeOf>().toEqualTypeOf() +const insert = SQLite.compileUnsafe(SQLite.insertInto(Items).values({ id: 1, note: "n" }).returning("note")) +expectTypeOf>>().toEqualTypeOf>() +// @ts-expect-error a select cannot name a missing column +SQLite.from(Items).select("missing") +// @ts-expect-error required params remain required through the SQLite compile wrapper +SQLite.compile(query, {}) +// @ts-expect-error an integer column cannot take a string +SQLite.insertInto(Items).values({ id: "wrong", note: "n" }) diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index 303848e..cfac13e 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -122,3 +122,27 @@ it("rolls back failed SQLite transactions and nested savepoints", async () => { })) expect(ids).toEqual([{ id: 1 }, { id: 3 }]) }) + +it("executes SQLite upserts, updates and deletes with RETURNING", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values([{ id: 1, note: "first", enabled: true, score: 2 }, { id: 2, note: "second", enabled: false, score: null }])) + const upsert = yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "replacement", enabled: true, score: 3 }) + .onConflictDoUpdate({ target: ["id"], set: { note: "updated" } }).returning("id", "note")) + const update = yield* db.run(SQLite.update(Items).set({ note: "changed" }).where(($) => [$.id.eq(2)]).returning("note")) + const deleted = yield* db.run(SQLite.deleteFrom(Items).where(($) => [$.id.eq(2)]).returning("id")) + return { upsert, update, deleted } + })) + expect(rows).toEqual({ upsert: [{ id: 1, note: "updated" }], update: [{ note: "changed" }], deleted: [{ id: 2 }] }) +}) + +it("runs scoped CTEs, joins and derived-table queries with the same row decoder", async () => { + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.run(SQLite.insertInto(Items).values({ id: 1, note: "one", enabled: true, score: null })) + const filtered = SQLite.from(Items).select("id", "note").where(($) => [$.id.eq(SQLite.param.int("id"))]) + const Recent = SQLite.table("recent", { id: SQLite.integer, note: SQLite.text }) + const cte = yield* db.run(SQLite.from(Recent).withCTE("recent", filtered).select(), { id: 1 }) + const joined = yield* db.run(SQLite.fromQuery(filtered, "filtered").innerJoin(Items, "i", (a, b) => a.id.eq(b.id)).select(($) => ({ id: $.id, note: $.i.note })), { id: 1 }) + return { cte, joined } + })) + expect(rows).toEqual({ cte: [{ id: 1, note: "one" }], joined: [{ id: 1, note: "one" }] }) +}) diff --git a/src/sqlite/types.ts b/src/sqlite/types.ts index ea6fd18..a28d9c5 100644 --- a/src/sqlite/types.ts +++ b/src/sqlite/types.ts @@ -11,7 +11,7 @@ export const integer = custom("integer", Schema.Int) export const real = custom("real", Schema.Finite) /** SQLite stores booleans as 0/1. Other integers are not booleans. */ export const boolean = custom("integer", Schema.BooleanFromBit) -/** An explicitly zoned instant stored as ISO text. */ +/** An instant stored as ISO text. */ export const timestamp = custom("text", Schema.DateTimeUtcFromString, PgTimestampLiteral) const bytes = Schema.Array(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 255 }))).pipe( diff --git a/tests/package-consumer-declarations.mts b/tests/package-consumer-declarations.mts index 616c47b..e4a191d 100644 --- a/tests/package-consumer-declarations.mts +++ b/tests/package-consumer-declarations.mts @@ -47,3 +47,8 @@ export const Recent = CH.table("recent", { external: true, columns: { name: CH.s export const withRecent = CH.from(Recent).withCTE("recent", eventsByApp).select(($) => ({ name: $.name })) export const fromSub = CH.fromQuery(eventsByApp, "sub").select(($) => ({ name: $.name })) export const both = CH.unionAll(eventsByApp, CH.from(Events).select(($) => ({ name: $.name }))) + +import * as SQLite from "@maple-dev/effect-orm/sqlite" +export const SqliteItems = SQLite.table("sqlite_items", { id: SQLite.integer, label: SQLite.text }) +export const sqliteById = SQLite.from(SqliteItems).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +export const sqliteCompile = () => SQLite.compile(sqliteById, { id: 1 }) diff --git a/tests/package-consumer.mts b/tests/package-consumer.mts index 5360120..6c62a7b 100644 --- a/tests/package-consumer.mts +++ b/tests/package-consumer.mts @@ -91,3 +91,14 @@ assert.equal(typeof runCli, "function") assert.equal(typeof Db.fromSqlClient, "function") assert.equal(Db.isContention(new Db.DatabaseError({ message: "x", sql: "", reason: "SerializationError", cause: undefined })), true) + +// SQLite works from the tarball without installing any SQLite driver. +const SQLite = await import("@maple-dev/effect-orm/sqlite") +const sqliteRows = SQLite.table("consumer_rows", { id: SQLite.integer, note: SQLite.text }) +const sqliteQuery = SQLite.from(sqliteRows).select().where(($) => [$.id.eq(SQLite.param.int("id"))]) +const sqliteCompiled = await Effect.runPromise(SQLite.compile(sqliteQuery, { id: 1 })) +assert.deepEqual(sqliteCompiled.parameters, [1]) +assert.equal(sqliteCompiled.dialect, "sqlite") +const sqliteDecoded: ReadonlyArray<{ readonly id: number; readonly note: string }> = await Effect.runPromise(sqliteCompiled.decodeRows([{ id: 1, note: "sqlite" }])) +assert.deepEqual(sqliteDecoded, [{ id: 1, note: "sqlite" }]) +assert.equal(SQLite.compileUnsafe(sqliteQuery, { id: 1 }, { dialect: SQLite.d1Dialect }).dialect, "d1") diff --git a/tsdown.config.ts b/tsdown.config.ts index 7e1f75c..0815e4c 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ expr: "./src/expr.ts", sql: "./src/sql/index.ts", postgres: "./src/postgres.ts", + sqlite: "./src/sqlite.ts", schema: "./src/schema.ts", tinybird: "./src/tinybird.ts", migrate: "./src/migrate.ts", From 0e96f312bdb8bdea6d29ebbc87979cae0faafc58 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:12:29 +0200 Subject: [PATCH 08/24] docs: list SQLite under ON CONFLICT in the reference --- docs/reference.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index d6c67c2..4bb4975 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -8,7 +8,7 @@ import * as PG from "@maple-dev/effect-orm/postgres" import * as SQLite from "@maple-dev/effect-orm/sqlite" ``` -All carry the whole [query builder](#query-builder-both-entries). Each adds its own column +All carry the whole [query builder](#query-builder-every-entry). Each adds its own column types, functions, table definitions and a `compile` that defaults to its dialect: [`/clickhouse`](#clickhouse) and [`/postgres`](#postgres). Everything else is on a [subpath](#other-subpaths) for a narrower job. @@ -33,7 +33,7 @@ uniformly, which some codebases prefer for exactly this reason. --- -## Query builder (both entries) +## Query builder (every entry) Everything in this section is exported, identically, from `/clickhouse`, `/postgres` and `/sqlite`. @@ -47,7 +47,7 @@ Everything in this section is exported, identically, from `/clickhouse`, `/postg | `unionAll` | `(...queries) => CHUnionQuery` | | `update` | `(table) => CHUpdateStart`, then `CHUpdate`: `.set(record \| fn)`, `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)`. See [Updating and deleting](./updates-and-deletes.md) | | `deleteFrom` | `(table) => CHDelete`: `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)` | -| `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres). See [Inserting rows](./inserts.md) | +| `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres and SQLite). See [Inserting rows](./inserts.md) | Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or [`PG.table`](#tables-and-ddl-1). From bfe02f87bc96ebf6ce95e2ff94dfa2cdee486598 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 09:45:59 +0200 Subject: [PATCH 09/24] fix(sqlite): build external tables from their columns --- .changeset/quiet-sqlite-dialects.md | 4 +-- docs/reference.md | 1 + src/sqlite.ts | 38 +++++++++++++++++++++++++++-- src/sqlite/sqlite.test-d.ts | 4 +++ src/sqlite/sqlite.test.ts | 7 ++++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md index c4b380e..628a175 100644 --- a/.changeset/quiet-sqlite-dialects.md +++ b/.changeset/quiet-sqlite-dialects.md @@ -4,5 +4,5 @@ Add the `/sqlite` query entry with SQLite codecs, functions and positional bindings. Use `sqliteDialect` with an Effect SQLite SqlClient, or `d1Dialect` for Cloudflare D1's binding limit -and explicit refusal of interactive transactions. SQLite schema/migration generation is not -implemented yet. +and explicit refusal of interactive transactions. `table` is a query schema only: SQLite DDL and +migration generation are not implemented yet. diff --git a/docs/reference.md b/docs/reference.md index 4bb4975..4d5e6ed 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -548,6 +548,7 @@ interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and | `json(schema)` | JSON text decoded through the supplied schema | | `nullable(type)`, `custom`, `brand` | Type composition | | `table(name, columns, options?)`, `TableOptions` | A query schema, including `tenantColumn`, defaults and computed-column metadata; no generated DDL | +| `table(name, ExternalTableDefinition)`, `ExternalTableDefinition` | `{ external: true, columns, tenantColumn? }`, the same query schema in the `/clickhouse` and `/postgres` form | | `count`, `countDistinct`, `countIf`, `sum`, `avg`, `min`, `max` | SQLite aggregates; the last four can return SQL NULL | | `lower`, `upper`, `length`, `coalesce` | SQLite scalar functions | diff --git a/src/sqlite.ts b/src/sqlite.ts index 4f04a9a..d8c0df2 100644 --- a/src/sqlite.ts +++ b/src/sqlite.ts @@ -1,12 +1,46 @@ -// SQLite queries and codecs. DDL/migrations stay with the caller for now. +// @maple-dev/effect-orm/sqlite +// +// SQLite and Cloudflare D1 queries, codecs, functions and execution through an +// Effect SqlClient. `table` is a query schema only: this entry writes no DDL +// and generates no migrations yet. See docs/sqlite.md. import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" import { compiled, compiledUnsafe } from "./ch/failure" +import { table as queryTable, type Table, type TableOptions } from "./ch/table" +import type { ColumnDefs } from "./ch/types" import { sqliteDialect } from "./sqlite/dialect" export * from "./core" export * from "./sqlite/types" export * from "./sqlite/functions" -export { table, type TableOptions } from "./ch/table" +export type { TableOptions } + +/** A table another tool owns, as on `/clickhouse` and `/postgres`. */ +export interface ExternalTableDefinition { + readonly external: true + readonly columns: Columns + readonly tenantColumn?: keyof Columns & string +} + +/** + * A SQLite query schema: `table(name, columns, options?)`, or + * `table(name, { external: true, columns, tenantColumn? })`. Neither carries DDL. + */ +export function table( + name: Name, + definition: ExternalTableDefinition, +): Table +export function table< + const Name extends string, + const Columns extends ColumnDefs, + const Defaulted extends keyof Columns & string = never, + const Computed extends keyof Columns & string = never, +>(name: Name, columns: Columns, options?: TableOptions): Table +export function table(name: string, definition: any, options?: TableOptions): Table { + if (options === undefined && definition.external === true && typeof definition.columns === "object") { + return queryTable(name, definition.columns, definition.tenantColumn === undefined ? undefined : { tenantColumn: definition.tenantColumn }) + } + return queryTable(name, definition, options) +} export { sqliteDialect, d1Dialect } from "./sqlite/dialect" /** The shared compiler, defaulting to SQLite. Pass d1Dialect for D1's limits. */ diff --git a/src/sqlite/sqlite.test-d.ts b/src/sqlite/sqlite.test-d.ts index aa2f74d..7d3f78a 100644 --- a/src/sqlite/sqlite.test-d.ts +++ b/src/sqlite/sqlite.test-d.ts @@ -16,3 +16,7 @@ SQLite.from(Items).select("missing") SQLite.compile(query, {}) // @ts-expect-error an integer column cannot take a string SQLite.insertInto(Items).values({ id: "wrong", note: "n" }) +const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer } }) +SQLite.from(External).select("id") +// @ts-expect-error an external table has only its declared columns +SQLite.from(External).select("columns") diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index cfac13e..eef78c0 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -146,3 +146,10 @@ it("runs scoped CTEs, joins and derived-table queries with the same row decoder" })) expect(rows).toEqual({ cte: [{ id: 1, note: "one" }], joined: [{ id: 1, note: "one" }] }) }) + +it("builds an external table from its columns, not from the definition's keys", () => { + const External = SQLite.table("ext", { external: true, columns: { id: SQLite.integer, org_id: SQLite.text }, tenantColumn: "org_id" }) + expect(Object.keys(External.columns)).toEqual(["id", "org_id"]) + expect(External.tenantColumn).toBe("org_id") + expect(SQLite.compileUnsafe(SQLite.from(External).select("id").where(($) => [$.org_id.eq("o")])).sql).toContain('"ext"."org_id" = ?') +}) From 02e48579762fd3e837adaaf9e9a913ffaa2fed94 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:02:13 +0100 Subject: [PATCH 10/24] fix(sqlite): portable LIKE escapes, no silent NUL truncation, exact fractions Found by running the fuzzer and the core builder suite on SQLite and D1: - LIKE has no default escape on SQLite, so `like("org\\_1")` matched a literal backslash and never `org_1`. The dialect now writes `ESCAPE '\'`, so `\` escapes `%`, `_` and `\` as on every dialect. - SQLite drivers end text at U+0000: node:sqlite reads `'a' || char(0)` back as `'a'`, bound or literal. A string with U+0000 is refused at compile time instead of being stored and read back cut short. (The hex-cast literal kept NUL in the SQL, but not in the value.) - SQLite parses some decimals to a neighbouring double, as a literal or through json_extract (`1.23456789e-95`). A fraction in an expression is bound instead of written as text; integers are still written. - `Amount > 9.75` was refused because SQLite.integer only encodes whole numbers. A fraction compared with an integer column now compiles, as on Postgres and ClickHouse; inserting one is still refused. docs/sqlite.md now states SQLite's ASCII-case-insensitive LIKE. Co-Authored-By: Claude Opus 5.5 --- docs/sqlite.md | 15 +++-- src/ch/compile.ts | 6 ++ src/ch/dialect.ts | 8 +++ src/ch/expr.ts | 38 +++++++++-- src/sql/sql-syntax.ts | 11 +++ src/sqlite/dialect.ts | 18 +++-- src/sqlite/sqlite.test.ts | 26 +++++-- tests/core.sqlite.test.ts | 26 +++++++ tests/fuzz/dialects/sqlite.ts | 123 ++++++++++++++++++++++++++++++++++ tests/sqlite-support.ts | 45 +++++++++++++ 10 files changed, 297 insertions(+), 19 deletions(-) create mode 100644 tests/core.sqlite.test.ts create mode 100644 tests/fuzz/dialects/sqlite.ts create mode 100644 tests/sqlite-support.ts diff --git a/docs/sqlite.md b/docs/sqlite.md index 2491ca9..3f6e115 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -103,13 +103,20 @@ an administrative read/write. Raw SQL passed to `query` or `execute` has no tena bypasses this guard. It is not a replacement for an application's authorization checks. The dialect handles OFFSET without LIMIT, branch-local ordering/limits in UNION ALL, and -INSERT SELECT with ON CONFLICT. It refuses these unsupported forms at compile time: - -- `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE` with SQLite's own case rules. +INSERT SELECT with ON CONFLICT. `LIKE` patterns escape `%`, `_` and `\` with `\` as on every +dialect: the builder writes `ESCAPE '\'`, which SQLite needs. A fraction is bound rather than +written as decimal text, because SQLite reads some decimals as a neighbouring double, and a +fraction compared with an `integer` column (`Amount > 9.75`) compiles as on Postgres. It refuses these unsupported forms at compile time: + +- `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE`, which on SQLite ignores ASCII case + (`'acme' LIKE 'ACME'` is true, unlike Postgres and ClickHouse). +- Text containing U+0000. SQLite stores it, but its drivers end text there, so the value would + read back cut short; the builder refuses it in literals and bound params. - `ON CONSTRAINT` conflict targets. Name the indexed columns instead. - Per-cell `DEFAULT` in a VALUES tuple. Omit a defaulted column from every row in the statement, or group rows by their present columns. All-default rows are not supported by the builder. - ClickHouse FORMAT/SETTINGS and built-in functions from another dialect. The tests run real SQLite through `@effect/sql-sqlite-node` and local workerd/D1 through -Miniflare. They require no deployed database or Cloudflare credentials. +Miniflare. They require no deployed database or Cloudflare credentials. Both run the shared core +builder suite (`tests/core.sqlite.test.ts`) and every fuzz property (`tests/fuzz`). diff --git a/src/ch/compile.ts b/src/ch/compile.ts index cb2a4e5..297daed 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -1549,6 +1549,12 @@ function renderParams( const value = literal !== undefined ? literal.wire : encodeParam(dialect, kind as ParamKind, name, params[name]) if (style._tag === "inline") return checkedLiteral(dialect, value, paramContext(kind, name)) + if (style.nulInText === false && typeof value === "string" && value.includes("\0")) { + return fail(new QueryBuilderError({ + code: "InvalidLiteral", + message: `compile: ${paramContext(kind, name)} contains U+0000, which ${dialect.name} drivers end text at; the value would arrive cut short`, + }), "NULL") + } const key = `${kind}\0${name}` const existing = style.reuse ? bound.get(key) : undefined if (existing !== undefined) return existing diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 095d7e5..26e4f8d 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -45,6 +45,12 @@ export type ParamStyle = * at the server or by being split into several statements. */ readonly maxParameters?: number + /** + * False where the driver ends text at U+0000 (SQLite's), so a bound + * string containing one would arrive cut short. Such a value fails to + * compile instead. + */ + readonly nulInText?: boolean } /** Clauses that exist in some dialects and not others. */ @@ -219,6 +225,8 @@ export const checkedLiteral = (dialect: Dialect, value: unknown, context: string const checkedSyntax = (dialect: Dialect): SqlSyntax => ({ dialect: dialect.name, ilike: dialect.ilike, + likeEscape: dialect.likeEscape, + inexactNumberText: dialect.inexactNumberText, quoteIdent: dialect.quoteIdent, quoteString: (value) => checked(dialect, dialect.quoteString(value), "a string literal"), literal: (value, context) => checkedLiteral(dialect, value, context), diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 34bb9f1..774b9d5 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -12,7 +12,7 @@ import { raw, str, ident, compile, as_ as sqlAs, known } from "../sql/sql-fragme import { activeSqlSyntax } from "../sql/sql-syntax" import { activeLiteralBinder } from "../sql/literal-binder" import { chDateTimeLiteral, CHFloatResult, CHNumber, string as chString, type CHType, type InferEncoded, type InferTS } from "./types" -import { encodeColumnLiteral, isSubnormal, numberText, sqlLiteral } from "./literal" +import { encodeColumnLiteral, encodeLiteral, isSubnormal, numberText, sqlLiteral } from "./literal" import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { fail } from "./failure" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" @@ -341,7 +341,9 @@ const describeUntyped = (value: unknown): string => * column at all. */ const comparisonLiteral = (columnType: CHType, value: unknown, column: string): string => { - const literal = encodeColumnLiteral(columnType, value, column) + const literal = sqliteFraction(columnType, value) + ? encodeLiteral(Schema.Finite, value, `column ${column}`) + : encodeColumnLiteral(columnType, value, column) // Outside a compile, ClickHouse is the default dialect. const dialect = activeSqlSyntax()?.dialect ?? "clickhouse" const bound = activeLiteralBinder() !== undefined @@ -357,11 +359,32 @@ const comparisonLiteral = (columnType: CHType, value: unknown, return literal } +/** + * A fraction compared with a SQLite `integer` column: `Amount > 9.75` is a + * meaningful comparison, but the column's codec only encodes whole numbers. + * SQLite compares an INTEGER with a REAL numerically, so the value is written + * as the number it is. Inserting a fraction is still refused by the codec. + */ +const sqliteFraction = (columnType: CHType, value: unknown): boolean => + columnType.sql === "integer" && + typeof value === "number" && + Number.isFinite(value) && + !Number.isInteger(value) && + (activeSqlSyntax()?.dialect === "sqlite" || activeSqlSyntax()?.dialect === "d1") + const POSTGRES_INTEGERS = new Set(["int2", "int4", "int8", "smallint", "integer", "bigint"]) /** A number as SQL that reads back exactly. A subnormal goes through the dialect: ClickHouse cannot parse one as written. */ -const numberFragment = (value: number): SqlFragment => - isSubnormal(value) ? known(() => (activeSqlSyntax()?.literal ?? sqlLiteral)(value, "a number literal")) : raw(numberText(value)) +const numberFragment = (value: number): SqlFragment => { + if (Number.isInteger(value)) return raw(numberText(value)) + return known(() => { + // SQLite reads some decimals back as a neighbouring double (`1.23456789e-95`), so a + // fraction is bound there, which the driver sends as the exact double. + const bind = activeSqlSyntax()?.inexactNumberText === true ? activeLiteralBinder() : undefined + if (bind !== undefined) return bind(value) + return isSubnormal(value) ? (activeSqlSyntax()?.literal ?? sqlLiteral)(value, "a number literal") : numberText(value) + }) +} const untypedLiteral = (value: boolean): string => activeSqlSyntax()?.literal(value, "an untyped boolean") ?? (value ? "1" : "0") @@ -387,6 +410,9 @@ const likePattern = (pattern: string | Expr): string => { ) } +/** `ESCAPE '\'` where LIKE has no default escape, so `\%` means a literal `%` on every dialect. */ +const likeEscape = (): string => (activeSqlSyntax()?.likeEscape === true ? " ESCAPE '\\'" : "") + const dateTimeLiteral = (value: DateTime.Utc): string => activeSqlSyntax()?.dateTimeLiteral(value) ?? compile(str(chDateTimeLiteral(value))) @@ -531,8 +557,8 @@ export function makeExpr( notBetween: (low, high) => makeCond(known(() => `${compile(fragment)} NOT BETWEEN ${compile(operand(low))} AND ${compile(operand(high))}`)), - like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}`)), - notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}`)), + like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}${likeEscape()}`)), + notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}${likeEscape()}`)), ilike: (pattern) => makeCond(known(() => activeSqlSyntax()?.ilike === false ? fail(new QueryBuilderDefect({ message: `ILIKE is not supported by the ${activeSqlSyntax()?.dialect} dialect; use LIKE with SQLite's own case rules` }), "FALSE") : `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index 89106c7..1bdd78a 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -13,6 +13,17 @@ export interface SqlSyntax { readonly dialect: string /** False when the dialect has no ILIKE operator. */ readonly ilike?: boolean + /** + * True when LIKE has no default escape character, so `\` in a pattern is + * literal unless the statement says `ESCAPE '\'` (SQLite). Patterns use `\` + * to escape `%`, `_` and `\` on every dialect. + */ + readonly likeEscape?: boolean + /** + * True when the database parses some decimal literals to a neighbouring + * double (SQLite), so a fraction must be bound rather than written as text. + */ + readonly inexactNumberText?: boolean /** One identifier (a column, alias, table or schema name), quoted as needed. */ readonly quoteIdent: (name: string) => string /** A string as a quoted literal. */ diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index 748e507..9c630f6 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -8,11 +8,17 @@ import { PgTimestampLiteral, timestampLiteral } from "../pg/types" const hex = (value: Uint8Array): string => Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("") -/** Hex text keeps NUL and the compiler's reserved param marker out of SQL text. */ +/** + * Hex text keeps the compiler's reserved param marker out of SQL text. U+0000 + * is refused: SQLite stores it, but its drivers end text there, so the value + * would read back cut short. + */ const quoteString = (value: string): string => - value.includes(PARAM_MARKER_PREFIX) || value.includes("\0") - ? `CAST(X'${hex(new TextEncoder().encode(value))}' AS TEXT)` - : `'${value.replace(/'/g, "''")}'` + value.includes("\0") + ? fail(new QueryBuilderError({ code: "InvalidLiteral", message: "a string literal contains U+0000, which SQLite drivers end text at; the value would read back cut short" }), "''") + : value.includes(PARAM_MARKER_PREFIX) + ? `CAST(X'${hex(new TextEncoder().encode(value))}' AS TEXT)` + : `'${value.replace(/'/g, "''")}'` const literal = (value: unknown, context: string): string => { if (value === null) return "NULL" @@ -28,11 +34,13 @@ export const sqliteDialect: Dialect = { name: "sqlite", functions: "sqlite", ilike: false, + likeEscape: true, + inexactNumberText: true, quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, quoteString, literal, dateTimeLiteral: (value) => quoteString(DateTime.formatIso(value)), - params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766 }, + params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766, nulInText: false }, clauses: { format: false, derivedTableAlias: true, diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index eef78c0..98a8988 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -85,9 +85,9 @@ it("round-trips JSON, blobs and zoned timestamps, and runs SQLite scalar functio expect(seen.scalar).toEqual([{ lower: "hello", upper: "HELLO", length: 5, fallback: 9 }]) }) -it("binds repeated positional params independently and quotes reserved-marker/NUL literals safely", async () => { +it("binds repeated positional params independently and quotes reserved-marker literals safely", async () => { const result = await withDatabase((db) => Effect.gen(function* () { - const note = "' OR 1=1 -- __PARAM_note__\0😀" + const note = "' OR 1=1 -- __PARAM_note__😀" yield* db.run(SQLite.insertInto(Items).values({ id: 1, note, enabled: false, score: null })) const query = SQLite.from(Items).select("id").where(($) => [$.note.eq(SQLite.param.string("note")), $.note.eq(SQLite.param.string("note"))]) const compiled = yield* SQLite.compile(query, { note }) @@ -95,9 +95,27 @@ it("binds repeated positional params independently and quotes reserved-marker/NU const literal = yield* db.query({ sql: `SELECT ${SQLite.sqliteDialect.quoteString(note)} AS note` }) return { parameters: compiled.parameters, rows, literal } })) - expect(result.parameters).toEqual(["' OR 1=1 -- __PARAM_note__\0😀", "' OR 1=1 -- __PARAM_note__\0😀"]) + expect(result.parameters).toEqual(["' OR 1=1 -- __PARAM_note__😀", "' OR 1=1 -- __PARAM_note__😀"]) expect(result.rows).toEqual([{ id: 1 }]) - expect(result.literal).toEqual([{ note: "' OR 1=1 -- __PARAM_note__\0😀" }]) + expect(result.literal).toEqual([{ note: "' OR 1=1 -- __PARAM_note__😀" }]) +}) + +it("refuses U+0000 in text, which SQLite drivers would cut short", () => { + // node:sqlite returns `SELECT 'a' || char(0) || 'b'` as "a": the value would be silently truncated. + const refused = (exit: Exit.Exit) => Exit.isFailure(exit) && /U\+0000/.test(String(Cause.squash(exit.cause))) + const where = SQLite.from(Items).select("id").where(($) => [$.note.eq(SQLite.param.string("note"))]) + expect(refused(Effect.runSyncExit(SQLite.compile(where, { note: "a\0b" })))).toBe(true) + expect(refused(Effect.runSyncExit(SQLite.compile(SQLite.insertInto(Items).values({ id: 1, note: "a\0b", enabled: true, score: null }))))).toBe(true) + expect(refused(Effect.runSyncExit(SQLite.compile(SQLite.from(Items).select(() => ({ v: SQLite.lit("a\0b") })))))).toBe(true) +}) + +it("binds a fraction instead of writing decimal text SQLite would read as another double", async () => { + const value = 1.23456789e-95 + const rows = await withDatabase((db) => db.run(SQLite.from(Items).select(() => ({ v: SQLite.lit(value) })).limit(1).crossTenant())) + const compiled = SQLite.compileUnsafe(SQLite.from(Items).select(() => ({ v: SQLite.lit(value) }))) + expect(compiled.sql).not.toContain("e-95") + expect(compiled.parameters).toEqual([value]) + expect(rows).toEqual([]) }) it("rejects a stored value that does not match the derived row schema", async () => { diff --git a/tests/core.sqlite.test.ts b/tests/core.sqlite.test.ts new file mode 100644 index 0000000..04d6852 --- /dev/null +++ b/tests/core.sqlite.test.ts @@ -0,0 +1,26 @@ +// The core builder suite on SQLite (node:sqlite, in-process) and D1 (a local +// workerd binding through Miniflare), so both run on every `vitest run`. +import { afterAll, describe, expect, it } from "@effect/vitest" +import { Effect } from "effect" +import { contexts, coreCases, coreSkips, expectedFor } from "./core-cases" +import { d1Database, sqliteDatabase } from "./sqlite-support" + +const databases = { sqlite: sqliteDatabase(), d1: d1Database() } +afterAll(() => Promise.all(Object.values(databases).map((db) => db.close()))) + +for (const dialect of ["sqlite", "d1"] as const) { + describe(`core builder suite on ${dialect}`, () => { + for (const fixture of coreCases) { + if (fixture.rejects?.[dialect] || Object.hasOwn(coreSkips[dialect], fixture.id)) continue + it.effect(fixture.id, () => + Effect.gen(function* () { + const compiled = fixture.build(contexts[dialect]) + if (fixture.metadata) expect(compiled).toMatchObject(fixture.metadata) + const wire = yield* Effect.promise(() => databases[dialect].query(compiled.sql, compiled.parameters)) + const rows = yield* compiled.decodeRows(wire) + expect(rows, compiled.sql).toEqual(expectedFor(fixture, dialect)) + }), + ) + } + }) +} diff --git a/tests/fuzz/dialects/sqlite.ts b/tests/fuzz/dialects/sqlite.ts new file mode 100644 index 0000000..71e4ed1 --- /dev/null +++ b/tests/fuzz/dialects/sqlite.ts @@ -0,0 +1,123 @@ +// SQLite (node:sqlite, in-process) and D1 (a local workerd binding through +// Miniflare). D1 executes SQLite's SQL, so both share semantics and literals; +// they differ in the dialect they compile with and the database they run on. +import fc from "fast-check" +import { DateTime, Schema } from "effect" +import * as SQLite from "@maple-dev/effect-orm/sqlite" +import { d1Context, sqliteContext } from "../../core-cases" +import { d1Database, sqliteDatabase } from "../../sqlite-support" +import { finite, nastyString, nullOr, utc, type LiteralType } from "../values" +import { runWith, type DialectModule, type FuzzDialect, type Semantics, type StringRules } from "./types" + +const semantics: Semantics = { + missingJoinColumn: "null", + integerDivision: "truncate", + modulo: "integer", + // NULL is SQLite's smallest value. + nulls: { asc: "first", desc: "last" }, + emptyAggregate: "null", + // Documented SQLite behaviour (docs/sqlite.md); the builder writes LIKE as is. + likeIgnoresCase: true, +} + +// Literals + +// SQLite stores U+0000, but its drivers end text there, so the dialect refuses it. +const strings: StringRules = { nul: false, leadingBom: true } +// The range DateTime.formatIso writes as `YYYY-…`, so text order is time order. +const MIN_TS = Date.UTC(0, 0, 1) +const MAX_TS = Date.UTC(9999, 11, 31, 23, 59, 59, 999) + +const hex = (bytes: Uint8Array) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("") + +/** + * A double as SQLite SQL that is exact without parsing a decimal: SQLite reads + * some decimals (`1.23456789e-95`, as a literal or through json_extract) as a + * neighbouring double. The integer mantissa is exact, and multiplying or + * dividing by a power of two below 2^62 is exact while the result is a double. + */ +const exactReal = (value: number): string => { + if (value === 0) return `SELECT 0.0 AS "v"` + const view = new DataView(new ArrayBuffer(8)) + view.setFloat64(0, Math.abs(value)) + const bits = view.getBigUint64(0) + const biased = Number(bits >> 52n) + const fraction = bits & ((1n << 52n) - 1n) + // value = mantissa * 2^exponent + const mantissa = biased === 0 ? fraction : fraction | (1n << 52n) + let exponent = (biased === 0 ? 1 : biased) - 1075 + let sql = `CAST(${value < 0 ? "-" : ""}${mantissa} AS REAL)` + while (exponent !== 0) { + const step = Math.min(Math.abs(exponent), 62) + sql += `${exponent > 0 ? " * " : " / "}${2n ** BigInt(step)}` + exponent -= Math.sign(exponent) * step + } + return `SELECT ${sql} AS "v"` +} + +/** The reference row, read by SQLite's JSON functions rather than written by the builder. */ +const jsonSource = (read: (value: string) => string) => (json: string) => + `SELECT ${read(`json_extract('${json.replaceAll("'", "''")}', '$.v')`)} AS "v"` + +const literals = (dialect: string): ReadonlyArray => { + const nasty = nastyString(strings) + const literal = ( + id: string, + type: SQLite.SqliteType, + value: fc.Arbitrary, + options: { wire?: (value: unknown) => unknown; read?: (value: string) => string; source?: (value: unknown) => string; comparable?: boolean } = {}, + ): LiteralType => ({ + id: `${dialect}:${id}`, + type, + value, + wire: options.wire ?? ((v) => v), + source: (json, v) => (options.source ? options.source(v) : jsonSource(options.read ?? ((x) => x))(json)), + comparable: options.comparable ?? true, + }) + return [ + literal("text", SQLite.text, nasty), + literal("integer", SQLite.integer, fc.maxSafeInteger()), + literal("real", SQLite.real, finite, { source: (v) => exactReal(v as number) }), + literal("boolean", SQLite.boolean, fc.boolean()), + literal("timestamp", SQLite.timestamp, fc.integer({ min: MIN_TS, max: MAX_TS }).map(utc), { + wire: (value) => DateTime.formatIso(value as DateTime.Utc), + }), + literal("blob", SQLite.blob, fc.uint8Array({ maxLength: 8 }), { wire: (value) => hex(value as Uint8Array), read: (v) => `unhex(${v})` }), + literal("json", SQLite.json(Schema.Struct({ message: Schema.String })), fc.record({ message: nasty }), { + wire: (value) => JSON.stringify(value), + comparable: false, + }), + literal("nullable(text)", SQLite.nullable(SQLite.text), nullOr(nasty)), + literal("nullable(integer)", SQLite.nullable(SQLite.integer), nullOr(fc.maxSafeInteger())), + ] +} + +const sqliteDb = sqliteDatabase() +const d1Db = d1Database() + +export const sqlite: FuzzDialect = { + name: "sqlite", + module: SQLite as never, + context: sqliteContext, + strings, + ilike: false, + literals: literals("sqlite"), + targets: [{ name: "sqlite", available: true, semantics, run: (compiled) => runWith(compiled, sqliteDb.query) }], + close: sqliteDb.close, +} + +const d1Module: DialectModule = { + ...(SQLite as never as DialectModule), + compile: (query, params) => SQLite.compile(query, params, { dialect: SQLite.d1Dialect }), +} + +export const d1: FuzzDialect = { + name: "d1", + module: d1Module, + context: d1Context, + strings, + ilike: false, + literals: literals("d1"), + targets: [{ name: "d1", available: true, semantics, run: (compiled) => runWith(compiled, d1Db.query) }], + close: d1Db.close, +} diff --git a/tests/sqlite-support.ts b/tests/sqlite-support.ts new file mode 100644 index 0000000..599065a --- /dev/null +++ b/tests/sqlite-support.ts @@ -0,0 +1,45 @@ +// Raw SQLite and D1 connections for the core suite and the fuzzer: no Effect +// SqlClient in between, so what the database returns is what the query decodes. +import { DatabaseSync } from "node:sqlite" +import { Miniflare } from "miniflare" + +export type RawQuery = (sql: string, parameters: ReadonlyArray) => Promise>> + +/** An in-memory SQLite (Node's built-in driver, the one `@effect/sql-sqlite-node` uses). */ +export const sqliteDatabase = (): { readonly query: RawQuery; readonly close: () => Promise } => { + const db = new DatabaseSync(":memory:") + return { + // node:sqlite rows have a null prototype; spread them so `toEqual` compares plain objects. + query: async (sql, parameters) => db.prepare(sql).all(...(parameters as Array)).map((row) => ({ ...row })), + close: async () => db.close(), + } +} + +/** A local D1 database in workerd, through the Workers binding an application gets. */ +export const d1Database = (): { readonly query: RawQuery; readonly close: () => Promise } => { + let started: Promise<{ readonly mf: Miniflare; readonly db: D1DatabaseLike }> | undefined + const start = () => + (started ??= (async () => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('fuzz') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + return { mf, db: (await mf.getD1Database("DB")) as unknown as D1DatabaseLike } + })()) + return { + query: async (sql, parameters) => { + const { db } = await start() + return (await db.prepare(sql).bind(...parameters).all()).results + }, + close: async () => { + if (started !== undefined) await (await started).mf.dispose() + started = undefined + }, + } +} + +interface D1DatabaseLike { + prepare(sql: string): { bind(...values: ReadonlyArray): { all(): Promise<{ results: Array> }> } } +} From 0216fdab83daf17e6ceb1498553035b9416b2253 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:02:13 +0100 Subject: [PATCH 11/24] test(sqlite): run the core builder suite and every fuzz property on SQLite and D1 - tests/core.sqlite.test.ts runs every shared core case on node:sqlite and on a local D1 binding (Miniflare), with SQL snapshots for both. - tests/fuzz/dialects/sqlite.ts registers `sqlite` and `d1` with the fuzzer: query plans against the reference model, literal round-trips for every SQLite column type, compile invariants and hostile values. Their semantics: NULL first ascending, truncating integer division, `%` on integers, ASCII-case-insensitive LIKE. - The real reference row is built from the double's mantissa and powers of two, since SQLite cannot parse some decimals exactly. - New core cases pin LIKE escapes and case on every dialect; ILIKE moves to its own case so `patterns-and-lists` also runs on SQLite. - Counterexamples pinned in tests/fuzz/regressions.ts. Co-Authored-By: Claude Opus 5.5 --- tests/__snapshots__/core-sql.test.ts.snap | 1703 +++++++++++++++++++-- tests/core-cases.ts | 108 +- tests/core-sql.test.ts | 2 +- tests/fuzz/dialects/index.ts | 3 +- tests/fuzz/regressions.ts | 13 +- 5 files changed, 1697 insertions(+), 132 deletions(-) diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index fba9e28..b3e36ed 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -240,6 +240,24 @@ SELECT } `; +exports[`core SQL (clickhouse) > ilike 1`] = ` +{ + "parameters": [], + "sql": "WITH orders AS ( + +), +customers AS ( + +) +SELECT + orders.Id AS id + FROM orders + WHERE orders.OrgId = 'org_1' + AND orders.Status ILIKE 'PA%' + ORDER BY id ASC", +} +`; + exports[`core SQL (clickhouse) > inner-join 1`] = ` { "parameters": [], @@ -340,6 +358,42 @@ SELECT } `; +exports[`core SQL (clickhouse) > like-case 1`] = ` +{ + "parameters": [], + "sql": "WITH orders AS ( + +), +customers AS ( + +) +SELECT + orders.Id AS id + FROM orders + WHERE orders.OrgId = 'org_1' + AND orders.Customer LIKE 'ACME' + ORDER BY id ASC", +} +`; + +exports[`core SQL (clickhouse) > like-escape 1`] = ` +{ + "parameters": [], + "sql": "WITH orders AS ( + +), +customers AS ( + +) +SELECT + orders.Id AS id + FROM orders + WHERE orders.OrgId LIKE 'org\\\\_1' + AND orders.Note NOT LIKE 'it\\\\_s' + ORDER BY id ASC", +} +`; + exports[`core SQL (clickhouse) > limit-offset 1`] = ` { "parameters": [], @@ -435,7 +489,7 @@ SELECT FROM orders WHERE orders.OrgId = 'org_1' AND (orders.Customer LIKE '%e%' AND orders.Customer NOT LIKE 'i%') - AND (orders.Status ILIKE 'PA%' OR orders.Status IN ('open')) + AND (orders.Status LIKE 'pa%' OR orders.Status IN ('open')) AND orders.Id NOT IN (3) AND NOT (orders.Id = 2) ORDER BY id ASC", @@ -604,7 +658,7 @@ FORMAT JSON", } `; -exports[`core SQL (postgres) > and-or 1`] = ` +exports[`core SQL (d1) > and-or 1`] = ` { "parameters": [ "org_1", @@ -621,13 +675,13 @@ exports[`core SQL (postgres) > and-or 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND (("orders"."Status" = $2 AND "orders"."Amount" > $3) OR "orders"."Customer" = $4) + WHERE "orders"."OrgId" = ? + AND (("orders"."Status" = ? AND "orders"."Amount" > ?) OR "orders"."Customer" = ?) ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > arithmetic 1`] = ` +exports[`core SQL (d1) > arithmetic 1`] = ` { "parameters": [ "org_1", @@ -646,12 +700,12 @@ SELECT "orders"."Amount" % 3 AS "remainder", "orders"."Amount" / 2 AS "divided" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Id" = $2", + WHERE "orders"."OrgId" = ? + AND "orders"."Id" = ?", } `; -exports[`core SQL (postgres) > comparisons 1`] = ` +exports[`core SQL (d1) > comparisons 1`] = ` { "parameters": [ "org_1", @@ -659,6 +713,7 @@ exports[`core SQL (postgres) > comparisons 1`] = ` 5, 10, 20, + 10, ], "sql": "WITH "orders" AS ( @@ -669,16 +724,16 @@ exports[`core SQL (postgres) > comparisons 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" != $2 - AND "orders"."Amount" > $3 - AND "orders"."Amount" >= $4 - AND "orders"."Amount" < $5 - AND "orders"."Amount" <= $4", + WHERE "orders"."OrgId" = ? + AND "orders"."Status" != ? + AND "orders"."Amount" > ? + AND "orders"."Amount" >= ? + AND "orders"."Amount" < ? + AND "orders"."Amount" <= ?", } `; -exports[`core SQL (postgres) > cross-join 1`] = ` +exports[`core SQL (d1) > cross-join 1`] = ` { "parameters": [ "org_1", @@ -696,16 +751,17 @@ SELECT "c"."Name" AS "customer" FROM "orders" CROSS JOIN "customers" AS "c" - WHERE "orders"."OrgId" = $1 - AND "c"."OrgId" = $2 - AND "orders"."Id" <= $3 + WHERE "orders"."OrgId" = ? + AND "c"."OrgId" = ? + AND "orders"."Id" <= ? ORDER BY "id" ASC, "customer" ASC", } `; -exports[`core SQL (postgres) > cross-join-subquery 1`] = ` +exports[`core SQL (d1) > cross-join-subquery 1`] = ` { "parameters": [ + "org_1", "org_1", 2, ], @@ -728,14 +784,14 @@ SELECT SELECT count(*) AS "customers" FROM "customers" - WHERE "customers"."OrgId" = $1) AS "k" - WHERE "orders"."OrgId" = $1 - AND "orders"."Id" <= $2 + WHERE "customers"."OrgId" = ?) AS "k" + WHERE "orders"."OrgId" = ? + AND "orders"."Id" <= ? ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > cte 1`] = ` +exports[`core SQL (d1) > cte 1`] = ` { "parameters": [ "org_1", @@ -758,8 +814,8 @@ SELECT "orders"."Customer" AS "Customer", "orders"."Amount" AS "Amount" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" = $2 + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? ) SELECT sum("paid"."Amount") AS "total" @@ -767,27 +823,7 @@ SELECT } `; -exports[`core SQL (postgres) > distinct 1`] = ` -{ - "parameters": [ - "org_1", - ], - "sql": "WITH "orders" AS ( - -), -"customers" AS ( - -) -SELECT DISTINCT ON ("customer") - "orders"."Customer" AS "customer", - "orders"."Id" AS "id" - FROM "orders" - WHERE "orders"."OrgId" = $1 - ORDER BY "customer" ASC, "id" DESC", -} -`; - -exports[`core SQL (postgres) > distinct-plain 1`] = ` +exports[`core SQL (d1) > distinct-plain 1`] = ` { "parameters": [ "org_1", @@ -801,12 +837,12 @@ exports[`core SQL (postgres) > distinct-plain 1`] = ` SELECT DISTINCT "orders"."Status" AS "status" FROM "orders" - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? ORDER BY "status" ASC", } `; -exports[`core SQL (postgres) > from-subquery 1`] = ` +exports[`core SQL (d1) > from-subquery 1`] = ` { "parameters": [ "org_1", @@ -826,14 +862,14 @@ SELECT "orders"."Customer" AS "customer", sum("orders"."Amount") AS "total" FROM "orders" - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? GROUP BY 1, 2) AS "t" - WHERE "t"."total" > $2 + WHERE "t"."total" > ? ORDER BY "total" DESC", } `; -exports[`core SQL (postgres) > group-having 1`] = ` +exports[`core SQL (d1) > group-having 1`] = ` { "parameters": [ "paid", @@ -848,19 +884,19 @@ exports[`core SQL (postgres) > group-having 1`] = ` SELECT "orders"."Customer" AS "customer", count(*) AS "orders", - count(*) FILTER (WHERE "orders"."Status" = $1) AS "paid", + count(*) FILTER (WHERE "orders"."Status" = ?) AS "paid", sum("orders"."Amount") AS "total", min("orders"."Amount") AS "smallest", max("orders"."Amount") AS "largest" FROM "orders" - WHERE "orders"."OrgId" = $2 + WHERE "orders"."OrgId" = ? GROUP BY 1 HAVING count(*) >= 2 ORDER BY "customer" ASC", } `; -exports[`core SQL (postgres) > inner-join 1`] = ` +exports[`core SQL (d1) > inner-join 1`] = ` { "parameters": [ "org_1", @@ -876,12 +912,12 @@ SELECT "c"."Tier" AS "tier" FROM "orders" INNER JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > is-not-null 1`] = ` +exports[`core SQL (d1) > is-not-null 1`] = ` { "parameters": [ "org_1", @@ -895,17 +931,19 @@ exports[`core SQL (postgres) > is-not-null 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? AND "orders"."Note" IS NOT NULL ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > join-subqueries 1`] = ` +exports[`core SQL (d1) > join-subqueries 1`] = ` { "parameters": [ "org_1", "silver", + "org_1", + "org_1", ], "sql": "WITH "orders" AS ( @@ -928,8 +966,8 @@ SELECT "customers"."Name" AS "name", "customers"."Tier" AS "tier" FROM "customers" - WHERE "customers"."OrgId" = $1 - AND "customers"."Tier" != $2) AS "t" ON "customers"."Name" = "t"."name" + WHERE "customers"."OrgId" = ? + AND "customers"."Tier" != ?) AS "t" ON "customers"."Name" = "t"."name" LEFT JOIN (WITH "orders" AS ( ), @@ -940,14 +978,14 @@ SELECT "orders"."Customer" AS "customer", count(*) AS "orders" FROM "orders" - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? GROUP BY 1) AS "n" ON "customers"."Name" = "n"."customer" - WHERE "customers"."OrgId" = $1 + WHERE "customers"."OrgId" = ? ORDER BY "name" ASC", } `; -exports[`core SQL (postgres) > left-join 1`] = ` +exports[`core SQL (d1) > left-join 1`] = ` { "parameters": [ "org_1", @@ -964,16 +1002,17 @@ SELECT "c"."Tier" AS "tier" FROM "orders" LEFT JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") - WHERE "orders"."OrgId" = $1 - AND "orders"."Id" >= $2 + WHERE "orders"."OrgId" = ? + AND "orders"."Id" >= ? ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > limit-offset 1`] = ` +exports[`core SQL (d1) > like-case 1`] = ` { "parameters": [ "org_1", + "ACME", ], "sql": "WITH "orders" AS ( @@ -984,18 +1023,37 @@ exports[`core SQL (postgres) > limit-offset 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 - ORDER BY "id" DESC - LIMIT 2 - OFFSET 1", + WHERE "orders"."OrgId" = ? + AND "orders"."Customer" LIKE ? ESCAPE '\\' + ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > locking 1`] = ` +exports[`core SQL (d1) > like-escape 1`] = ` +{ + "parameters": [ + "org\\_1", + "it\\_s", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" LIKE ? ESCAPE '\\' + AND "orders"."Note" NOT LIKE ? ESCAPE '\\' + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (d1) > limit-offset 1`] = ` { "parameters": [ "org_1", - 1, ], "sql": "WITH "orders" AS ( @@ -1006,19 +1064,21 @@ exports[`core SQL (postgres) > locking 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Id" = $2 - FOR UPDATE SKIP LOCKED", + WHERE "orders"."OrgId" = ? + ORDER BY "id" DESC + LIMIT 2 + OFFSET 1", } `; -exports[`core SQL (postgres) > null-and-range 1`] = ` +exports[`core SQL (d1) > null-and-range 1`] = ` { "parameters": [ "org_1", 5, 20, 3, + 3, ], "sql": "WITH "orders" AS ( @@ -1029,15 +1089,15 @@ exports[`core SQL (postgres) > null-and-range 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 + WHERE "orders"."OrgId" = ? AND "orders"."Note" IS NULL - AND "orders"."Amount" BETWEEN $2 AND $3 - AND "orders"."Id" NOT BETWEEN $4 AND $4 + AND "orders"."Amount" BETWEEN ? AND ? + AND "orders"."Id" NOT BETWEEN ? AND ? ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > nulls-and-literals 1`] = ` +exports[`core SQL (d1) > nulls-and-literals 1`] = ` { "parameters": [ "org_1", @@ -1056,13 +1116,13 @@ SELECT coalesce("orders"."Note", '-') AS "shown", 'x''y' AS "label" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Id" IN ($2, $3) + WHERE "orders"."OrgId" = ? + AND "orders"."Id" IN (?, ?) ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > params 1`] = ` +exports[`core SQL (d1) > params 1`] = ` { "parameters": [ "org_1", @@ -1080,21 +1140,21 @@ SELECT "orders"."Id" AS "id", "orders"."Created" AS "created" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Amount" >= $2 - AND "orders"."Amount" < $3::float8 - AND "orders"."Created" >= $4::timestamptz + WHERE "orders"."OrgId" = ? + AND "orders"."Amount" >= ? + AND "orders"."Amount" < ? + AND "orders"."Created" >= ? ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > patterns-and-lists 1`] = ` +exports[`core SQL (d1) > patterns-and-lists 1`] = ` { "parameters": [ "org_1", "%e%", "i%", - "PA%", + "pa%", "open", 3, 2, @@ -1108,16 +1168,16 @@ exports[`core SQL (postgres) > patterns-and-lists 1`] = ` SELECT "orders"."Id" AS "id" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND ("orders"."Customer" LIKE $2 AND "orders"."Customer" NOT LIKE $3) - AND ("orders"."Status" ILIKE $4 OR "orders"."Status" IN ($5)) - AND "orders"."Id" NOT IN ($6) - AND NOT ("orders"."Id" = $7) + WHERE "orders"."OrgId" = ? + AND ("orders"."Customer" LIKE ? ESCAPE '\\' AND "orders"."Customer" NOT LIKE ? ESCAPE '\\') + AND ("orders"."Status" LIKE ? ESCAPE '\\' OR "orders"."Status" IN (?)) + AND "orders"."Id" NOT IN (?) + AND NOT ("orders"."Id" = ?) ORDER BY "id" ASC", } `; -exports[`core SQL (postgres) > route-and-cross-tenant 1`] = ` +exports[`core SQL (d1) > route-and-cross-tenant 1`] = ` { "parameters": [], "sql": "WITH "orders" AS ( @@ -1132,7 +1192,7 @@ SELECT } `; -exports[`core SQL (postgres) > select-columns 1`] = ` +exports[`core SQL (d1) > select-columns 1`] = ` { "parameters": [ "org_1", @@ -1148,16 +1208,16 @@ SELECT "orders"."Id" AS "Id", "orders"."Customer" AS "Customer" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" = $2 + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? ORDER BY "Id" ASC", } `; -exports[`core SQL (postgres) > selected-param 1`] = ` +exports[`core SQL (d1) > selected-param 1`] = ` { "parameters": [ - true, + 1, "org_1", 1, ], @@ -1169,23 +1229,25 @@ exports[`core SQL (postgres) > selected-param 1`] = ` ) SELECT "orders"."Id" AS "id", - $1::boolean AS "flagged" + ? AS "flagged" FROM "orders" - WHERE "orders"."OrgId" = $2 - AND "orders"."Id" = $3", + WHERE "orders"."OrgId" = ? + AND "orders"."Id" = ?", } `; -exports[`core SQL (postgres) > union 1`] = ` +exports[`core SQL (d1) > union 1`] = ` { "parameters": [ "org_1", "paid", + "org_1", "open", + "org_1", "void", ], "sql": "SELECT * FROM ( -( +SELECT * FROM ( WITH "orders" AS ( ), @@ -1196,12 +1258,12 @@ SELECT "orders"."Status" AS "status", sum("orders"."Amount") AS "total" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" = $2 + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? GROUP BY 1 -) +) AS "__union_branch" UNION ALL -( +SELECT * FROM ( WITH "orders" AS ( ), @@ -1212,12 +1274,12 @@ SELECT "orders"."Status" AS "status", sum("orders"."Amount") AS "total" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" = $3 + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? GROUP BY 1 -) +) AS "__union_branch" UNION ALL -( +SELECT * FROM ( WITH "orders" AS ( ), @@ -1228,10 +1290,10 @@ SELECT "orders"."Status" AS "status", sum("orders"."Amount") AS "total" FROM "orders" - WHERE "orders"."OrgId" = $1 - AND "orders"."Status" = $4 + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? GROUP BY 1 -) +) AS "__union_branch" ) AS "__union" ORDER BY "total" DESC LIMIT 2 @@ -1239,10 +1301,10 @@ OFFSET 1", } `; -exports[`core SQL (postgres) > union-cross-tenant 1`] = ` +exports[`core SQL (d1) > union-cross-tenant 1`] = ` { "parameters": [], - "sql": "( + "sql": "SELECT * FROM ( WITH "orders" AS ( ), @@ -1252,9 +1314,9 @@ WITH "orders" AS ( SELECT count(*) AS "orders" FROM "orders" -) +) AS "__union_branch" UNION ALL -( +SELECT * FROM ( WITH "orders" AS ( ), @@ -1264,6 +1326,1405 @@ WITH "orders" AS ( SELECT count(*) AS "orders" FROM "orders" -)", +) AS "__union_branch"", +} +`; + +exports[`core SQL (postgres) > and-or 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + 15, + "globex", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND (("orders"."Status" = $2 AND "orders"."Amount" > $3) OR "orders"."Customer" = $4) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > arithmetic 1`] = ` +{ + "parameters": [ + "org_1", + 4, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Amount" + 1 AS "added", + "orders"."Amount" - 1 AS "subtracted", + "orders"."Amount" * 2 AS "multiplied", + "orders"."Amount" % 3 AS "remainder", + "orders"."Amount" / 2 AS "divided" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" = $2", +} +`; + +exports[`core SQL (postgres) > comparisons 1`] = ` +{ + "parameters": [ + "org_1", + "void", + 5, + 10, + 20, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" != $2 + AND "orders"."Amount" > $3 + AND "orders"."Amount" >= $4 + AND "orders"."Amount" < $5 + AND "orders"."Amount" <= $4", +} +`; + +exports[`core SQL (postgres) > cross-join 1`] = ` +{ + "parameters": [ + "org_1", + "org_1", + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Name" AS "customer" + FROM "orders" + CROSS JOIN "customers" AS "c" + WHERE "orders"."OrgId" = $1 + AND "c"."OrgId" = $2 + AND "orders"."Id" <= $3 + ORDER BY "id" ASC, "customer" ASC", +} +`; + +exports[`core SQL (postgres) > cross-join-subquery 1`] = ` +{ + "parameters": [ + "org_1", + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "k"."customers" AS "customers" + FROM "orders" + CROSS JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "customers" + FROM "customers" + WHERE "customers"."OrgId" = $1) AS "k" + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" <= $2 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > cte 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +), +"paid" AS ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "Customer", + "orders"."Amount" AS "Amount" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" = $2 +) +SELECT + sum("paid"."Amount") AS "total" + FROM "paid"", +} +`; + +exports[`core SQL (postgres) > distinct 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT DISTINCT ON ("customer") + "orders"."Customer" AS "customer", + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + ORDER BY "customer" ASC, "id" DESC", +} +`; + +exports[`core SQL (postgres) > distinct-plain 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT DISTINCT + "orders"."Status" AS "status" + FROM "orders" + WHERE "orders"."OrgId" = $1 + ORDER BY "status" ASC", +} +`; + +exports[`core SQL (postgres) > from-subquery 1`] = ` +{ + "parameters": [ + "org_1", + 6, + ], + "sql": "SELECT + "t"."customer" AS "customer", + "t"."total" AS "total" + FROM (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."OrgId" AS "OrgId", + "orders"."Customer" AS "customer", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = $1 + GROUP BY 1, 2) AS "t" + WHERE "t"."total" > $2 + ORDER BY "total" DESC", +} +`; + +exports[`core SQL (postgres) > group-having 1`] = ` +{ + "parameters": [ + "paid", + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "customer", + count(*) AS "orders", + count(*) FILTER (WHERE "orders"."Status" = $1) AS "paid", + sum("orders"."Amount") AS "total", + min("orders"."Amount") AS "smallest", + max("orders"."Amount") AS "largest" + FROM "orders" + WHERE "orders"."OrgId" = $2 + GROUP BY 1 + HAVING count(*) >= 2 + ORDER BY "customer" ASC", +} +`; + +exports[`core SQL (postgres) > ilike 1`] = ` +{ + "parameters": [ + "org_1", + "PA%", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" ILIKE $2 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > inner-join 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Tier" AS "tier" + FROM "orders" + INNER JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") + WHERE "orders"."OrgId" = $1 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > is-not-null 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Note" IS NOT NULL + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > join-subqueries 1`] = ` +{ + "parameters": [ + "org_1", + "silver", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "customers"."Name" AS "name", + "t"."tier" AS "tier", + "n"."orders" AS "orders" + FROM "customers" + INNER JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "customers"."Name" AS "name", + "customers"."Tier" AS "tier" + FROM "customers" + WHERE "customers"."OrgId" = $1 + AND "customers"."Tier" != $2) AS "t" ON "customers"."Name" = "t"."name" + LEFT JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "customer", + count(*) AS "orders" + FROM "orders" + WHERE "orders"."OrgId" = $1 + GROUP BY 1) AS "n" ON "customers"."Name" = "n"."customer" + WHERE "customers"."OrgId" = $1 + ORDER BY "name" ASC", +} +`; + +exports[`core SQL (postgres) > left-join 1`] = ` +{ + "parameters": [ + "org_1", + 3, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Tier" AS "tier" + FROM "orders" + LEFT JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" >= $2 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > like-case 1`] = ` +{ + "parameters": [ + "org_1", + "ACME", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Customer" LIKE $2 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > like-escape 1`] = ` +{ + "parameters": [ + "org\\_1", + "it\\_s", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" LIKE $1 + AND "orders"."Note" NOT LIKE $2 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > limit-offset 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + ORDER BY "id" DESC + LIMIT 2 + OFFSET 1", +} +`; + +exports[`core SQL (postgres) > locking 1`] = ` +{ + "parameters": [ + "org_1", + 1, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" = $2 + FOR UPDATE SKIP LOCKED", +} +`; + +exports[`core SQL (postgres) > null-and-range 1`] = ` +{ + "parameters": [ + "org_1", + 5, + 20, + 3, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Note" IS NULL + AND "orders"."Amount" BETWEEN $2 AND $3 + AND "orders"."Id" NOT BETWEEN $4 AND $4 + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > nulls-and-literals 1`] = ` +{ + "parameters": [ + "org_1", + 2, + 4, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "orders"."Note" AS "note", + coalesce("orders"."Note", '-') AS "shown", + 'x''y' AS "label" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" IN ($2, $3) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > params 1`] = ` +{ + "parameters": [ + "org_1", + 7, + 19.5, + "2026-01-01T00:00:00.000Z", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "orders"."Created" AS "created" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Amount" >= $2 + AND "orders"."Amount" < $3::float8 + AND "orders"."Created" >= $4::timestamptz + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > patterns-and-lists 1`] = ` +{ + "parameters": [ + "org_1", + "%e%", + "i%", + "pa%", + "open", + 3, + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND ("orders"."Customer" LIKE $2 AND "orders"."Customer" NOT LIKE $3) + AND ("orders"."Status" LIKE $4 OR "orders"."Status" IN ($5)) + AND "orders"."Id" NOT IN ($6) + AND NOT ("orders"."Id" = $7) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > route-and-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders"", +} +`; + +exports[`core SQL (postgres) > select-columns 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "Id", + "orders"."Customer" AS "Customer" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" = $2 + ORDER BY "Id" ASC", +} +`; + +exports[`core SQL (postgres) > selected-param 1`] = ` +{ + "parameters": [ + true, + "org_1", + 1, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + $1::boolean AS "flagged" + FROM "orders" + WHERE "orders"."OrgId" = $2 + AND "orders"."Id" = $3", +} +`; + +exports[`core SQL (postgres) > union 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + "open", + "void", + ], + "sql": "SELECT * FROM ( +( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" = $2 + GROUP BY 1 +) +UNION ALL +( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" = $3 + GROUP BY 1 +) +UNION ALL +( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = $1 + AND "orders"."Status" = $4 + GROUP BY 1 +) +) AS "__union" +ORDER BY "total" DESC +LIMIT 2 +OFFSET 1", +} +`; + +exports[`core SQL (postgres) > union-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +) +UNION ALL +( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +)", +} +`; + +exports[`core SQL (sqlite) > and-or 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + 15, + "globex", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND (("orders"."Status" = ? AND "orders"."Amount" > ?) OR "orders"."Customer" = ?) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > arithmetic 1`] = ` +{ + "parameters": [ + "org_1", + 4, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Amount" + 1 AS "added", + "orders"."Amount" - 1 AS "subtracted", + "orders"."Amount" * 2 AS "multiplied", + "orders"."Amount" % 3 AS "remainder", + "orders"."Amount" / 2 AS "divided" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Id" = ?", +} +`; + +exports[`core SQL (sqlite) > comparisons 1`] = ` +{ + "parameters": [ + "org_1", + "void", + 5, + 10, + 20, + 10, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" != ? + AND "orders"."Amount" > ? + AND "orders"."Amount" >= ? + AND "orders"."Amount" < ? + AND "orders"."Amount" <= ?", +} +`; + +exports[`core SQL (sqlite) > cross-join 1`] = ` +{ + "parameters": [ + "org_1", + "org_1", + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Name" AS "customer" + FROM "orders" + CROSS JOIN "customers" AS "c" + WHERE "orders"."OrgId" = ? + AND "c"."OrgId" = ? + AND "orders"."Id" <= ? + ORDER BY "id" ASC, "customer" ASC", +} +`; + +exports[`core SQL (sqlite) > cross-join-subquery 1`] = ` +{ + "parameters": [ + "org_1", + "org_1", + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "k"."customers" AS "customers" + FROM "orders" + CROSS JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "customers" + FROM "customers" + WHERE "customers"."OrgId" = ?) AS "k" + WHERE "orders"."OrgId" = ? + AND "orders"."Id" <= ? + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > cte 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +), +"paid" AS ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "Customer", + "orders"."Amount" AS "Amount" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? +) +SELECT + sum("paid"."Amount") AS "total" + FROM "paid"", +} +`; + +exports[`core SQL (sqlite) > distinct-plain 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT DISTINCT + "orders"."Status" AS "status" + FROM "orders" + WHERE "orders"."OrgId" = ? + ORDER BY "status" ASC", +} +`; + +exports[`core SQL (sqlite) > from-subquery 1`] = ` +{ + "parameters": [ + "org_1", + 6, + ], + "sql": "SELECT + "t"."customer" AS "customer", + "t"."total" AS "total" + FROM (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."OrgId" AS "OrgId", + "orders"."Customer" AS "customer", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = ? + GROUP BY 1, 2) AS "t" + WHERE "t"."total" > ? + ORDER BY "total" DESC", +} +`; + +exports[`core SQL (sqlite) > group-having 1`] = ` +{ + "parameters": [ + "paid", + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "customer", + count(*) AS "orders", + count(*) FILTER (WHERE "orders"."Status" = ?) AS "paid", + sum("orders"."Amount") AS "total", + min("orders"."Amount") AS "smallest", + max("orders"."Amount") AS "largest" + FROM "orders" + WHERE "orders"."OrgId" = ? + GROUP BY 1 + HAVING count(*) >= 2 + ORDER BY "customer" ASC", +} +`; + +exports[`core SQL (sqlite) > inner-join 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Tier" AS "tier" + FROM "orders" + INNER JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") + WHERE "orders"."OrgId" = ? + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > is-not-null 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Note" IS NOT NULL + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > join-subqueries 1`] = ` +{ + "parameters": [ + "org_1", + "silver", + "org_1", + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "customers"."Name" AS "name", + "t"."tier" AS "tier", + "n"."orders" AS "orders" + FROM "customers" + INNER JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "customers"."Name" AS "name", + "customers"."Tier" AS "tier" + FROM "customers" + WHERE "customers"."OrgId" = ? + AND "customers"."Tier" != ?) AS "t" ON "customers"."Name" = "t"."name" + LEFT JOIN (WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Customer" AS "customer", + count(*) AS "orders" + FROM "orders" + WHERE "orders"."OrgId" = ? + GROUP BY 1) AS "n" ON "customers"."Name" = "n"."customer" + WHERE "customers"."OrgId" = ? + ORDER BY "name" ASC", +} +`; + +exports[`core SQL (sqlite) > left-join 1`] = ` +{ + "parameters": [ + "org_1", + 3, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Tier" AS "tier" + FROM "orders" + LEFT JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") + WHERE "orders"."OrgId" = ? + AND "orders"."Id" >= ? + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > like-case 1`] = ` +{ + "parameters": [ + "org_1", + "ACME", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Customer" LIKE ? ESCAPE '\\' + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > like-escape 1`] = ` +{ + "parameters": [ + "org\\_1", + "it\\_s", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" LIKE ? ESCAPE '\\' + AND "orders"."Note" NOT LIKE ? ESCAPE '\\' + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > limit-offset 1`] = ` +{ + "parameters": [ + "org_1", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + ORDER BY "id" DESC + LIMIT 2 + OFFSET 1", +} +`; + +exports[`core SQL (sqlite) > null-and-range 1`] = ` +{ + "parameters": [ + "org_1", + 5, + 20, + 3, + 3, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Note" IS NULL + AND "orders"."Amount" BETWEEN ? AND ? + AND "orders"."Id" NOT BETWEEN ? AND ? + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > nulls-and-literals 1`] = ` +{ + "parameters": [ + "org_1", + 2, + 4, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "orders"."Note" AS "note", + coalesce("orders"."Note", '-') AS "shown", + 'x''y' AS "label" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Id" IN (?, ?) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > params 1`] = ` +{ + "parameters": [ + "org_1", + 7, + 19.5, + "2026-01-01T00:00:00.000Z", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "orders"."Created" AS "created" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Amount" >= ? + AND "orders"."Amount" < ? + AND "orders"."Created" >= ? + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > patterns-and-lists 1`] = ` +{ + "parameters": [ + "org_1", + "%e%", + "i%", + "pa%", + "open", + 3, + 2, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND ("orders"."Customer" LIKE ? ESCAPE '\\' AND "orders"."Customer" NOT LIKE ? ESCAPE '\\') + AND ("orders"."Status" LIKE ? ESCAPE '\\' OR "orders"."Status" IN (?)) + AND "orders"."Id" NOT IN (?) + AND NOT ("orders"."Id" = ?) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > route-and-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders"", +} +`; + +exports[`core SQL (sqlite) > select-columns 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "Id", + "orders"."Customer" AS "Customer" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? + ORDER BY "Id" ASC", +} +`; + +exports[`core SQL (sqlite) > selected-param 1`] = ` +{ + "parameters": [ + 1, + "org_1", + 1, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + ? AS "flagged" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Id" = ?", +} +`; + +exports[`core SQL (sqlite) > union 1`] = ` +{ + "parameters": [ + "org_1", + "paid", + "org_1", + "open", + "org_1", + "void", + ], + "sql": "SELECT * FROM ( +SELECT * FROM ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? + GROUP BY 1 +) AS "__union_branch" +UNION ALL +SELECT * FROM ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? + GROUP BY 1 +) AS "__union_branch" +UNION ALL +SELECT * FROM ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Status" AS "status", + sum("orders"."Amount") AS "total" + FROM "orders" + WHERE "orders"."OrgId" = ? + AND "orders"."Status" = ? + GROUP BY 1 +) AS "__union_branch" +) AS "__union" +ORDER BY "total" DESC +LIMIT 2 +OFFSET 1", +} +`; + +exports[`core SQL (sqlite) > union-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "SELECT * FROM ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +) AS "__union_branch" +UNION ALL +SELECT * FROM ( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +) AS "__union_branch"", } `; diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 3698368..88dc9ab 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -8,10 +8,11 @@ import { DateTime } from "effect" import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" +import * as SQLite from "@maple-dev/effect-orm/sqlite" -export type DialectName = "clickhouse" | "postgres" +export type DialectName = "clickhouse" | "postgres" | "sqlite" | "d1" /** ClickHouse runs twice: `join_use_nulls=0` fills a missing join row with defaults. */ -export type Target = "clickhouse" | "clickhouse-join-nulls" | "postgres" +export type Target = "clickhouse" | "clickhouse-join-nulls" | "postgres" | "sqlite" | "d1" type Col = CH.CHType type Orders = { @@ -70,6 +71,9 @@ interface FixtureColumn { readonly clickhouse: string readonly postgres: string } +/** SQLite (and D1) store the fixture's timestamps as the ISO text the dialect writes. */ +const sqliteValue = (column: FixtureColumn, value: Value): Value => + column.name === "Created" && typeof value === "string" ? `${value.replace(" ", "T")}Z` : value const fixtureColumns = { orders: [ { name: "OrgId", clickhouse: "String", postgres: "text" }, @@ -116,6 +120,11 @@ export const fixtureSql = (dialect: DialectName, table: keyof typeof fixtureColu const structure = columns.map((column) => `${column.name} ${column.clickhouse}`).join(", ") return `SELECT * FROM values('${structure.replaceAll("'", "\\'")}', ${tuples})` } + if (dialect === "sqlite" || dialect === "d1") { + // SQLite names VALUES columns column1, column2, …, and has no column list on a derived table. + const rows = fixtureRows[table].map((row) => `(${row.map((v, i) => sqlValue(sqliteValue(columns[i]!, v))).join(", ")})`).join(", ") + return `SELECT ${columns.map((column, i) => `column${i + 1} AS "${column.name}"`).join(", ")} FROM (VALUES ${rows})` + } const casts = columns.map((column, i) => `CAST("c${i}" AS ${column.postgres}) AS "${column.name}"`).join(", ") const names = columns.map((_, i) => `"c${i}"`).join(", ") return `SELECT ${casts} FROM (VALUES ${tuples}) AS v(${names})` @@ -189,9 +198,55 @@ export const postgresContext: CoreContext = { compileUnion: (union, params = {}) => PG.compileUnionUnsafe(union, params), } +const sqliteTables = { + orders: SQLite.table("orders", { + external: true, + tenantColumn: "OrgId", + columns: { + OrgId: SQLite.text, + Id: SQLite.integer, + Customer: SQLite.text, + Amount: SQLite.integer, + Status: SQLite.text, + Note: SQLite.nullable(SQLite.text), + Created: SQLite.timestamp, + }, + }), + customers: SQLite.table("customers", { external: true, columns: { OrgId: SQLite.text, Name: SQLite.text, Tier: SQLite.text }, tenantColumn: "OrgId" }), +} +const sqliteFn: CoreContext["fn"] = { + count: () => SQLite.count(), + countIf: (condition) => SQLite.countIf(condition), + sum: (expr) => SQLite.sum(expr), + min: (expr) => SQLite.min(expr), + max: (expr) => SQLite.max(expr), + coalesce: (expr, fallback) => SQLite.coalesce(expr, fallback), +} + +export const sqliteContext: CoreContext = { + dialect: "sqlite", + ...sqliteTables, + from: withFixtures("sqlite"), + types: { text: SQLite.text, int: SQLite.integer }, + fn: sqliteFn, + compile: (query, params = {}) => SQLite.compileUnsafe(query, params), + compileUnion: (union, params = {}) => SQLite.compileUnionUnsafe(union, params), +} + +/** D1 runs SQLite's SQL, with its own limits (100 bindings, no interactive transactions). */ +export const d1Context: CoreContext = { + ...sqliteContext, + dialect: "d1", + from: withFixtures("d1"), + compile: (query, params = {}) => SQLite.compileUnsafe(query, params, { dialect: SQLite.d1Dialect }), + compileUnion: (union, params = {}) => SQLite.compileUnionUnsafe(union, params, { dialect: SQLite.d1Dialect }), +} + export const contexts: Record = { clickhouse: clickhouseContext, postgres: postgresContext, + sqlite: sqliteContext, + d1: d1Context, } // Cases @@ -247,7 +302,7 @@ export const coreCases: readonly CoreCase[] = [ }, { id: "patterns-and-lists", - covers: [...e("like", "notLike", "ilike", "in_", "notIn"), ...c("and", "or"), "function:not"], + covers: [...e("like", "notLike", "in_", "notIn"), ...c("and", "or"), "function:not"], build: (ctx) => ctx.compile( ctx @@ -256,7 +311,7 @@ export const coreCases: readonly CoreCase[] = [ .where(($) => [ $.OrgId.eq(CH.param.string("orgId")), $.Customer.like("%e%").and($.Customer.notLike("i%")), - $.Status.ilike("PA%").or($.Status.in_("open")), + $.Status.like("pa%").or($.Status.in_("open")), $.Id.notIn(3), CH.not($.Id.eq(2)), ]) @@ -266,6 +321,38 @@ export const coreCases: readonly CoreCase[] = [ // acme (1, 2) and globex (3) match the patterns; 3 and 2 are excluded by notIn and not. expected: [{ id: 1 }], }, + { + id: "ilike", + covers: e("ilike"), + build: (ctx) => + ctx.compile( + orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.Status.ilike("PA%")]).orderBy(["id", "asc"]), + org, + ), + expected: [{ id: 1 }, { id: 2 }], + rejects: { sqlite: /ILIKE/, d1: /ILIKE/ }, + }, + { + id: "like-escape", + covers: e("like"), + // `\_` is a literal underscore on every dialect: SQLite needs `ESCAPE '\'` for it. + build: (ctx) => + ctx.compile( + ctx.from(ctx.orders).select(($) => ({ id: $.Id })).where(($) => [$.OrgId.like("org\\_1"), $.Note.notLike("it\\_s")]).orderBy(["id", "asc"]), + {}, + ), + expected: [{ id: 1 }, { id: 4 }], + metadata: { tenantScope: "cross-tenant" }, + }, + { + id: "like-case", + covers: e("like"), + build: (ctx) => + ctx.compile(orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.Customer.like("ACME")]).orderBy(["id", "asc"]), org), + // SQLite's LIKE ignores ASCII case; every other dialect's matches it exactly. Documented in docs/sqlite.md. + expected: [], + expectedBy: { sqlite: [{ id: 1 }, { id: 2 }], d1: [{ id: 1 }, { id: 2 }] }, + }, { id: "arithmetic", covers: e("add", "sub", "mul", "mod", "div"), @@ -283,11 +370,13 @@ export const coreCases: readonly CoreCase[] = [ .where(($) => [$.OrgId.eq(CH.param.string("orgId")), $.Id.eq(4)]), org, ), - // `/` is float division on ClickHouse and integer division on two Postgres integers. + // `/` is float division on ClickHouse and integer division on two Postgres or SQLite integers. expectedBy: { clickhouse: [{ added: 8, subtracted: 6, multiplied: 14, remainder: 1, divided: 3.5 }], "clickhouse-join-nulls": [{ added: 8, subtracted: 6, multiplied: 14, remainder: 1, divided: 3.5 }], postgres: [{ added: 8, subtracted: 6, multiplied: 14, remainder: 1, divided: 3 }], + sqlite: [{ added: 8, subtracted: 6, multiplied: 14, remainder: 1, divided: 3 }], + d1: [{ added: 8, subtracted: 6, multiplied: 14, remainder: 1, divided: 3 }], }, }, { @@ -598,7 +687,7 @@ export const coreCases: readonly CoreCase[] = [ id: "format", covers: q("format"), format: "JSON", - rejects: { postgres: /no FORMAT clause/ }, + rejects: { postgres: /no FORMAT clause/, sqlite: /no FORMAT clause/, d1: /no FORMAT clause/ }, build: (ctx) => ctx.compile( orgOrders(ctx) @@ -613,7 +702,7 @@ export const coreCases: readonly CoreCase[] = [ id: "union-format", covers: u("format"), format: "JSON", - rejects: { postgres: /no FORMAT clause/ }, + rejects: { postgres: /no FORMAT clause/, sqlite: /no FORMAT clause/, d1: /no FORMAT clause/ }, build: (ctx) => { const one = (id: number) => orgOrders(ctx) @@ -674,6 +763,7 @@ export const coreCases: readonly CoreCase[] = [ .orderBy(["customer", "asc"], ["id", "desc"]), org, ), + rejects: { sqlite: /DISTINCT ON/, d1: /DISTINCT ON/ }, expected: [ { customer: "acme", id: 2 }, { customer: "globex", id: 3 }, @@ -690,7 +780,7 @@ export const coreCases: readonly CoreCase[] = [ { id: "locking", covers: q("forUpdate", "forNoKeyUpdate", "forShare", "forKeyShare"), - rejects: { clickhouse: /no row locks/ }, + rejects: { clickhouse: /no row locks/, sqlite: /no row locks/, d1: /no row locks/ }, build: (ctx) => { // Each strength compiles; the one sent is FOR UPDATE SKIP LOCKED. const base = orgOrders(ctx).select(($) => ({ id: $.Id })).where(($) => [$.Id.eq(1)]) @@ -705,4 +795,6 @@ export const coreCases: readonly CoreCase[] = [ export const coreSkips: Record> = { clickhouse: {}, postgres: {}, + sqlite: {}, + d1: {}, } diff --git a/tests/core-sql.test.ts b/tests/core-sql.test.ts index 676abf1..e430d37 100644 --- a/tests/core-sql.test.ts +++ b/tests/core-sql.test.ts @@ -9,7 +9,7 @@ const elideFixtures = (dialect: DialectName, sql: string) => .replaceAll(fixtureSql(dialect, "orders"), "") .replaceAll(fixtureSql(dialect, "customers"), "") -for (const dialect of ["clickhouse", "postgres"] as const) { +for (const dialect of ["clickhouse", "postgres", "sqlite", "d1"] as const) { describe(`core SQL (${dialect})`, () => { for (const fixture of coreCases) { it(fixture.id, () => { diff --git a/tests/fuzz/dialects/index.ts b/tests/fuzz/dialects/index.ts index 166513a..b036402 100644 --- a/tests/fuzz/dialects/index.ts +++ b/tests/fuzz/dialects/index.ts @@ -1,9 +1,10 @@ // The dialect registry. Every fuzz suite iterates it; add a dialect here. import { clickhouse } from "./clickhouse" import { postgres } from "./postgres" +import { d1, sqlite } from "./sqlite" import type { FuzzDialect, FuzzTarget } from "./types" -export const dialects: ReadonlyArray = [clickhouse, postgres] +export const dialects: ReadonlyArray = [clickhouse, postgres, sqlite, d1] /** Every target, with the dialect whose SQL it runs. */ export const targets: ReadonlyArray<{ readonly dialect: FuzzDialect; readonly target: FuzzTarget }> = dialects.flatMap((dialect) => diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index a21260e..c5618ba 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -36,7 +36,16 @@ export const pinned: Record> = { { k: "or", style: "method", cs: [{ k: "and", style: "method", cs: [{ k: "const", v: true }, { k: "tenant", side: "orders" }] }, { k: "tenant", side: "orders" }] }, ], }, - // pg-fraction-bound-to-integer. + // sqlite-like-no-default-escape: `\_` was a literal backslash on SQLite. + { + orgId: "org_2", + join: "left", + where: [{ k: "tenant", side: "orders" }], + shape: { k: "agg", keys: [], aggs: [["a0", { k: "countIf", c: { k: "like", op: "like", l: { k: "jcol", col: "OrgId" }, pattern: "org\\__" } }]], having: [], orderBy: [] }, + wrap: false, + crossTenant: false, + }, + // pg-fraction-bound-to-integer, and sqlite-integer-refuses-fraction. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), where: [{ k: "cmp", op: "gt", l: { k: "col", col: "Amount" }, r: { k: "value", v: 9.75 } }], @@ -48,6 +57,8 @@ export const pinned: Record> = { "clickhouse:Array(Nullable(Int64)) compare": [[0, 4294967296]], "clickhouse:String": ["__PARAM_string_x__", "\\'", "\0"], "postgres:boolean select": [false], + // SQLite reads these decimals as a neighbouring double, as a literal or through json_extract. + ":real ": [1.23456789e-95, 5.464465248610941e-277, -1.415281131826549e-267, 0, 5e-324], "postgres:text[] select": [[]], "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], From 236a1034bd6db6e223494f40bf21aa0017dc39da Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:41:07 +0100 Subject: [PATCH 12/24] fix(sqlite): timestamps in SQLite's own text form, refuse integers past 2^53 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SQLite.timestamp` stored and compared ISO text (`…T16:28:54.750Z`), while SQLite's functions write `YYYY-MM-DD HH:MM:SS`. SQLite compares text by bytes, and ' ' sorts before 'T', so a row a DEFAULT stamped just now was not later than an hour ago: wrong rows, no error. Timestamps now use SQLite's own form, `YYYY-MM-DD HH:MM:SS.SSS` in UTC (what strftime('%f') and datetime(..., 'subsec') write), for inserts, comparisons and params; both forms decode, and zoneless text is UTC. The literal fuzz's reference row is now written by SQLite's datetime(), so builder values must equal the database's own text. `sqliteTimestampText` is exported for raw SQL. `SQLite.integer` decoded any integer-valued number. D1 rounds an INTEGER past 2^53 to a different number, which then passed; it is a decode error now. Co-Authored-By: Claude Opus 5.5 --- .changeset/quiet-sqlite-dialects.md | 5 ++- docs/reference.md | 3 +- docs/sqlite.md | 13 ++++++- src/sqlite/d1.test.ts | 12 +++++++ src/sqlite/dialect.ts | 9 ++--- src/sqlite/sqlite.test.ts | 26 ++++++++++++++ src/sqlite/timestamp.ts | 44 +++++++++++++++++++++++ src/sqlite/types.ts | 23 +++++++++--- tests/__snapshots__/core-sql.test.ts.snap | 4 +-- tests/core-cases.ts | 6 ++-- tests/fuzz/dialects/sqlite.ts | 9 +++-- 11 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 src/sqlite/timestamp.ts diff --git a/.changeset/quiet-sqlite-dialects.md b/.changeset/quiet-sqlite-dialects.md index 628a175..32eaa74 100644 --- a/.changeset/quiet-sqlite-dialects.md +++ b/.changeset/quiet-sqlite-dialects.md @@ -4,5 +4,8 @@ Add the `/sqlite` query entry with SQLite codecs, functions and positional bindings. Use `sqliteDialect` with an Effect SQLite SqlClient, or `d1Dialect` for Cloudflare D1's binding limit -and explicit refusal of interactive transactions. `table` is a query schema only: SQLite DDL and +and explicit refusal of interactive transactions. `timestamp` stores SQLite's own +`YYYY-MM-DD HH:MM:SS.SSS` UTC text, so it compares correctly with what `strftime` writes; `LIKE` +escapes with `\` as on every dialect; text with U+0000 and integers past 2^53 are refused rather +than silently changed. `table` is a query schema only: SQLite DDL and migration generation are not implemented yet. diff --git a/docs/reference.md b/docs/reference.md index 4d5e6ed..f7cd842 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -543,7 +543,8 @@ interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and | `SqliteType` | A SQLite SQL type with its Effect codec | | `text`, `integer`, `real` | Text, safe integer and finite number codecs | | `boolean` | A boolean encoded as 0/1; other integers fail decoding | -| `timestamp` | An instant stored as ISO text | +| `timestamp` | An instant stored as SQLite's `YYYY-MM-DD HH:MM:SS.SSS` UTC text; reads ISO-8601 too | +| `sqliteTimestampText(epochMillis)` | That text for an instant, for raw SQL and seeds | | `blob` | Bytes from a Uint8Array or D1's array wire representation | | `json(schema)` | JSON text decoded through the supplied schema | | `nullable(type)`, `custom`, `brand` | Type composition | diff --git a/docs/sqlite.md b/docs/sqlite.md index 3f6e115..05bc3fd 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -29,7 +29,18 @@ Unlike the ClickHouse and Postgres entries, SQLite has no schema diff, introspec ledger or kit dialect yet. A `table` value here cannot be passed to the DDL generators. The entry exports `text`, `integer` (a safe JavaScript integer), `real` (finite numbers), -`boolean` (0/1), `blob`, `timestamp` (ISO text), `json(schema)`, `nullable`, `custom` and `brand`. +`boolean` (0/1), `blob`, `timestamp`, `json(schema)`, `nullable`, `custom` and `brand`. + +`integer` refuses a stored value past 2^53 with a decode error: as a JavaScript number it would +no longer be the stored integer (D1 rounds it; node:sqlite refuses it). Use `custom` with a +bigint codec for such columns. + +`timestamp` stores an instant as SQLite's own text form, `YYYY-MM-DD HH:MM:SS.SSS` in UTC, the +form `strftime('%Y-%m-%d %H:%M:%f')` and `datetime(..., 'subsec')` write. It reads that form +and ISO-8601; text without a zone is UTC. SQLite compares text by bytes, so a timestamp column +orders by time only when every value in it has one form: write them through the builder or with +`strftime('%Y-%m-%d %H:%M:%f', ...)`. `CURRENT_TIMESTAMP` writes no milliseconds +(`2026-01-01 00:00:00`), which compares as earlier than the same instant written with them. `blob` accepts the Uint8Array wire form from Node and the array-of-bytes form from D1. `json` always uses the supplied schema to validate decoded JSON. Use `text` when a stored date/time must remain a string. diff --git a/src/sqlite/d1.test.ts b/src/sqlite/d1.test.ts index ff9c123..c35df5a 100644 --- a/src/sqlite/d1.test.ts +++ b/src/sqlite/d1.test.ts @@ -86,3 +86,15 @@ it("decodes D1 JSON and blob wire values and returns typed errors for malformed if (!Exit.isFailure(seen.malformed)) throw new Error("malformed JSON was accepted") expect(Cause.findError(seen.malformed.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) }, 30_000) + +it("refuses a D1 integer past 2^53 instead of decoding a different number", async () => { + const Big = SQLite.table("big", { id: SQLite.integer }) + const exit = await withD1((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE big (id INTEGER)" }) + // D1 reads 9007199254740993 as the JS number 9007199254740992: no longer the stored value. + yield* db.execute({ sql: "INSERT INTO big VALUES (9007199254740993)" }) + return yield* Effect.exit(db.run(SQLite.from(Big).select().crossTenant())) + })) + if (!Exit.isFailure(exit)) throw new Error(`an unsafe integer decoded as ${JSON.stringify(exit.value)}`) + expect(Cause.findError(exit.cause)).toMatchObject({ success: { _tag: "@maple-dev/effect-orm/CompiledQueryDecodeError" } }) +}, 30_000) diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index 9c630f6..ccaef3f 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -3,7 +3,8 @@ import { noTransactions, type Dialect } from "../ch/dialect" import { QueryBuilderError } from "../ch/errors" import { fail } from "../ch/failure" import { PARAM_MARKER_PREFIX } from "../ch/param" -import { PgTimestampLiteral, timestampLiteral } from "../pg/types" +import { timestampLiteral } from "../pg/types" +import { SqliteTimestampLiteral, sqliteTimestampText } from "./timestamp" const hex = (value: Uint8Array): string => Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("") @@ -39,7 +40,7 @@ export const sqliteDialect: Dialect = { quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, quoteString, literal, - dateTimeLiteral: (value) => quoteString(DateTime.formatIso(value)), + dateTimeLiteral: (value) => quoteString(sqliteTimestampText(DateTime.toEpochMillis(value))), params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766, nulInText: false }, clauses: { format: false, @@ -56,8 +57,8 @@ export const sqliteDialect: Dialect = { distinctOn: false, }, paramCodecs: { - dateTime: PgTimestampLiteral, - dateTimeSeconds: timestampLiteral((millis) => new Date(Math.floor(millis / 1000) * 1000).toISOString()), + dateTime: SqliteTimestampLiteral, + dateTimeSeconds: timestampLiteral((millis) => sqliteTimestampText(Math.floor(millis / 1000) * 1000)), }, transactions: { support: "full", diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index 98a8988..1ba4e77 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -100,6 +100,32 @@ it("binds repeated positional params independently and quotes reserved-marker li expect(result.literal).toEqual([{ note: "' OR 1=1 -- __PARAM_note__😀" }]) }) +it("compares timestamps with the text SQLite's own functions write, and reads zoneless text as UTC", async () => { + const Events = SQLite.table("events", { id: SQLite.integer, at: SQLite.timestamp }) + const rows = await withDatabase((db) => Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE events (id INTEGER, at TEXT NOT NULL)" }) + // Written by SQLite, not the builder: strftime's %f form, and CURRENT_TIMESTAMP-style text read back. + yield* db.execute({ sql: "INSERT INTO events VALUES (1, strftime('%Y-%m-%d %H:%M:%f', 'now')), (2, '2026-01-01 00:00:00')" }) + const recent = yield* db.run(SQLite.from(Events).select("id").where(($) => [$.at.gt(DateTime.makeUnsafe(Date.now() - 3_600_000))])) + const zoneless = yield* db.run(SQLite.from(Events).select("at").where(($) => [$.id.eq(2)])) + return { recent, zoneless } + })) + expect(rows.recent).toEqual([{ id: 1 }]) + // No zone in the text means UTC, whatever the host's time zone. + expect(rows.zoneless.map((row) => DateTime.formatIso(row.at))).toEqual(["2026-01-01T00:00:00.000Z"]) + const written = SQLite.compileUnsafe(SQLite.insertInto(Events).values({ id: 3, at: DateTime.makeUnsafe("2026-10-01T10:00:00.5+02:00") })) + expect(written.parameters).toContain("2026-10-01 08:00:00.500") +}) + +it("refuses an integer past 2^53 instead of decoding a different number", async () => { + const exit = await withDatabase((db) => Effect.gen(function* () { + // node:sqlite refuses an unsafe integer itself; D1 hands one over (see d1.test.ts), where the codec refuses it. + yield* db.execute({ sql: "INSERT INTO items VALUES (9007199254740993, 'big', 1, NULL)" }) + return yield* Effect.exit(db.run(SQLite.from(Items).select("id"))) + })) + expect(Exit.isFailure(exit) ? String(Cause.squash(exit.cause)) : exit.value).toMatch(/safe integer|too large to be represented/) +}) + it("refuses U+0000 in text, which SQLite drivers would cut short", () => { // node:sqlite returns `SELECT 'a' || char(0) || 'b'` as "a": the value would be silently truncated. const refused = (exit: Exit.Exit) => Exit.isFailure(exit) && /U\+0000/.test(String(Cause.squash(exit.cause))) diff --git a/src/sqlite/timestamp.ts b/src/sqlite/timestamp.ts new file mode 100644 index 0000000..e30b08f --- /dev/null +++ b/src/sqlite/timestamp.ts @@ -0,0 +1,44 @@ +// SQLite timestamp text: the form `SQLite.timestamp` writes and reads. +import { DateTime, Schema, SchemaGetter } from "effect" +import { pgTimestampToIso, timestampLiteral } from "../pg/types" + +/** + * An instant as SQLite's own text form, `YYYY-MM-DD HH:MM:SS.SSS` in UTC: what + * `strftime('%Y-%m-%d %H:%M:%f')` and `datetime(..., 'subsec')` write. SQLite + * compares text by bytes, so a column only orders by time when every value has + * one form; this is the form SQLite's own functions produce (see `timestampNow`). + */ +export const sqliteTimestampText = (epochMillis: number): string => { + const iso = new Date(epochMillis).toISOString() + return `${iso.slice(0, 10)} ${iso.slice(11, 23)}` +} + +/** Years SQLite's date functions and the text form cover: four digits, no sign. */ +const inSqliteRange = Schema.makeFilter( + (value: DateTime.Utc) => { + const year = new Date(DateTime.toEpochMillis(value)).getUTCFullYear() + return year >= 0 && year <= 9999 ? undefined : `year ${year} is outside 0000-9999, which SQLite's text timestamps cover` + }, + { title: "sqliteTimestampRange" }, +) + +const isTimestampText = Schema.makeFilter( + (value: string) => + Number.isNaN(Date.parse(pgTimestampToIso(value))) + ? `\`${value}\` is not a timestamp: expected \`YYYY-MM-DD HH:MM:SS[.SSS]\` or ISO-8601` + : undefined, + { title: "sqliteTimestamp" }, +) + +/** Reads SQLite's form and ISO-8601 (zoneless text is UTC); writes SQLite's form. */ +export const SqliteTimestamp: Schema.Codec = Schema.String.pipe( + Schema.check(isTimestampText), + Schema.decodeTo(Schema.DateTimeUtc.pipe(Schema.check(inSqliteRange)), { + decode: SchemaGetter.transform((value: string) => DateTime.makeUnsafe(pgTimestampToIso(value))), + encode: SchemaGetter.transform((value: DateTime.Utc) => sqliteTimestampText(DateTime.toEpochMillis(value))), + }), +) + +/** What a timestamp comparison or param accepts (a `DateTime.Utc`, a `Date`, timestamp text), written in SQLite's form. */ +export const SqliteTimestampLiteral = timestampLiteral(sqliteTimestampText) + diff --git a/src/sqlite/types.ts b/src/sqlite/types.ts index a28d9c5..9fb9990 100644 --- a/src/sqlite/types.ts +++ b/src/sqlite/types.ts @@ -1,18 +1,31 @@ import { Schema, SchemaGetter } from "effect" import { brand, custom, type CHType } from "../ch/types" -import { PgTimestampLiteral } from "../pg/types" +import { SqliteTimestamp, SqliteTimestampLiteral } from "./timestamp" + +export { sqliteTimestampText } from "./timestamp" /** A SQLite storage type and the codec for its wire values. */ export type SqliteType = CHType export const text = custom("text", Schema.String) -/** A JavaScript safe integer; use a custom codec for a driver's bigint mode. */ -export const integer = custom("integer", Schema.Int) +/** + * Past 2^53 a driver's number is no longer the stored integer (node:sqlite and + * D1 both read INTEGER as a JS number), so a value outside the safe range is a + * decode error, not a silently different number. Use a custom codec for a + * driver's bigint mode. + */ +const SafeInt = Schema.Number.check( + Schema.makeFilter((value: number) => (Number.isSafeInteger(value) ? undefined : `${value} is not a safe integer; SQLite integers past 2^53 lose precision as JS numbers`), { + title: "safeInteger", + }), +) +/** A JavaScript safe integer; see `SafeInt`. */ +export const integer = custom("integer", SafeInt) export const real = custom("real", Schema.Finite) /** SQLite stores booleans as 0/1. Other integers are not booleans. */ export const boolean = custom("integer", Schema.BooleanFromBit) -/** An instant stored as ISO text. */ -export const timestamp = custom("text", Schema.DateTimeUtcFromString, PgTimestampLiteral) +/** An instant stored as SQLite timestamp text; see `sqliteTimestampText`. */ +export const timestamp = custom("text", SqliteTimestamp, SqliteTimestampLiteral) const bytes = Schema.Array(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 255 }))).pipe( Schema.decodeTo(Schema.Uint8Array, { diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index b3e36ed..943bc1f 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -1128,7 +1128,7 @@ exports[`core SQL (d1) > params 1`] = ` "org_1", 7, 19.5, - "2026-01-01T00:00:00.000Z", + "2026-01-01 00:00:00.000", ], "sql": "WITH "orders" AS ( @@ -2527,7 +2527,7 @@ exports[`core SQL (sqlite) > params 1`] = ` "org_1", 7, 19.5, - "2026-01-01T00:00:00.000Z", + "2026-01-01 00:00:00.000", ], "sql": "WITH "orders" AS ( diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 88dc9ab..cbe809a 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -71,9 +71,6 @@ interface FixtureColumn { readonly clickhouse: string readonly postgres: string } -/** SQLite (and D1) store the fixture's timestamps as the ISO text the dialect writes. */ -const sqliteValue = (column: FixtureColumn, value: Value): Value => - column.name === "Created" && typeof value === "string" ? `${value.replace(" ", "T")}Z` : value const fixtureColumns = { orders: [ { name: "OrgId", clickhouse: "String", postgres: "text" }, @@ -122,7 +119,8 @@ export const fixtureSql = (dialect: DialectName, table: keyof typeof fixtureColu } if (dialect === "sqlite" || dialect === "d1") { // SQLite names VALUES columns column1, column2, …, and has no column list on a derived table. - const rows = fixtureRows[table].map((row) => `(${row.map((v, i) => sqlValue(sqliteValue(columns[i]!, v))).join(", ")})`).join(", ") + // The fixture's timestamps are already SQLite's text form, `YYYY-MM-DD HH:MM:SS.SSS`. + const rows = fixtureRows[table].map((row) => `(${row.map(sqlValue).join(", ")})`).join(", ") return `SELECT ${columns.map((column, i) => `column${i + 1} AS "${column.name}"`).join(", ")} FROM (VALUES ${rows})` } const casts = columns.map((column, i) => `CAST("c${i}" AS ${column.postgres}) AS "${column.name}"`).join(", ") diff --git a/tests/fuzz/dialects/sqlite.ts b/tests/fuzz/dialects/sqlite.ts index 71e4ed1..84bad67 100644 --- a/tests/fuzz/dialects/sqlite.ts +++ b/tests/fuzz/dialects/sqlite.ts @@ -24,7 +24,7 @@ const semantics: Semantics = { // SQLite stores U+0000, but its drivers end text there, so the dialect refuses it. const strings: StringRules = { nul: false, leadingBom: true } -// The range DateTime.formatIso writes as `YYYY-…`, so text order is time order. +// The years SQLite's text timestamps and date functions cover. const MIN_TS = Date.UTC(0, 0, 1) const MAX_TS = Date.UTC(9999, 11, 31, 23, 59, 59, 999) @@ -79,8 +79,13 @@ const literals = (dialect: string): ReadonlyArray => { literal("integer", SQLite.integer, fc.maxSafeInteger()), literal("real", SQLite.real, finite, { source: (v) => exactReal(v as number) }), literal("boolean", SQLite.boolean, fc.boolean()), + // The reference row is SQLite's own text: datetime() writes the date and time, so a builder + // value compares equal to what the database itself writes, not just to itself. literal("timestamp", SQLite.timestamp, fc.integer({ min: MIN_TS, max: MAX_TS }).map(utc), { - wire: (value) => DateTime.formatIso(value as DateTime.Utc), + source: (value) => { + const ms = DateTime.toEpochMillis(value as DateTime.Utc) + return `SELECT datetime(${Math.floor(ms / 1000)}, 'unixepoch') || '.' || printf('%03d', ${((ms % 1000) + 1000) % 1000}) AS "v"` + }, }), literal("blob", SQLite.blob, fc.uint8Array({ maxLength: 8 }), { wire: (value) => hex(value as Uint8Array), read: (v) => `unhex(${v})` }), literal("json", SQLite.json(Schema.Struct({ message: Schema.String })), fc.record({ message: nasty }), { From 8184ac422f0beafd432b6739784132dc2c29a474 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 18:45:52 +0100 Subject: [PATCH 13/24] test(fuzz): fuzz unions and writes on every dialect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unions: `unionAll` of two or three row plans that select the same aliases (each alias one sort in every branch), each branch with its own join, filter, DISTINCT, ORDER BY and LIMIT, under the union's ORDER BY, LIMIT and OFFSET. The model concatenates the branches' rows and pages them; the union pages only when it orders by every alias, so ties are equal rows. New properties `unions match the model (, …)`. Writes: a new suite. Each input gets a fresh scratch table on the dialect's first target and runs a sequence of builder writes: multi-row inserts leaving defaulted or nullable columns out, INSERT … SELECT, ON CONFLICT DO NOTHING / DO UPDATE (with the existing and the excluded row), UPDATE and DELETE with a filter or allRows(), and RETURNING. After every write the table is read back and compared with a JS model of its rows, and RETURNING with the rows the write touched. A write the dialect does not support is still generated and must be refused at compile time. `FuzzDialect.writes` declares the capabilities and the scratch table (query `Table`, create/drop DDL, `exec`); `DialectModule` gains unionAll, compileUnion, insertInto, update and deleteFrom. ClickHouse targets pass mutations_sync=2 and lightweight_deletes_sync=2 so an ALTER TABLE UPDATE or DELETE is visible to the next read. Found: a ClickHouse 26 UNION ALL column whose branches have numeric types with no common supertype (UInt64 and Int64, Int64 and Float64) becomes a Variant, and ORDER BY on it fails. Recorded in known-issues (clickhouse-union-variant-order-by) and pinned. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/README.md | 11 +- tests/fuzz/build.ts | 39 +++- tests/fuzz/dialects/clickhouse.ts | 29 +++ tests/fuzz/dialects/postgres.ts | 20 ++ tests/fuzz/dialects/types.ts | 39 +++- tests/fuzz/known-issues.ts | 39 +++- tests/fuzz/model.ts | 21 +- tests/fuzz/plan.ts | 82 ++++++- tests/fuzz/queries.fuzz.test.ts | 51 ++++- tests/fuzz/regressions.ts | 16 +- tests/fuzz/writes.fuzz.test.ts | 351 ++++++++++++++++++++++++++++++ 11 files changed, 667 insertions(+), 31 deletions(-) create mode 100644 tests/fuzz/writes.fuzz.test.ts diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 25a6b4f..4c34553 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -17,7 +17,8 @@ and are skipped otherwise. Postgres runs in-process on PGlite. | File | Property | Oracle | | --- | --- | --- | -| `queries.fuzz.test.ts` | random plans return the rows `model.ts` computes | a JS model of SQL, per target | +| `queries.fuzz.test.ts` | random plans, and `unionAll`s of them (branch and union ORDER BY / LIMIT / OFFSET), return the rows `model.ts` computes | a JS model of SQL, per target | +| `writes.fuzz.test.ts` | random sequences of inserts (multi-row, defaulted cells, INSERT … SELECT, ON CONFLICT), updates and deletes, with RETURNING, leave a scratch table as the model says; unsupported writes are refused | a JS model of the table's rows | | `literals.fuzz.test.ts` | every column type's values survive as a literal and as a param | the database's own JSON parser | | `compile.fuzz.test.ts` | plans compile deterministically, with no leaked JS or misaligned placeholders; hostile values are refused with a typed error | invariants | | `registry.fuzz.test.ts` | every `Dialect` in `src/` is registered here | — | @@ -36,6 +37,14 @@ A `FuzzDialect` gives: - `strings` and `ilike`: what the generators may produce for it. - `literals`: its column types, each with a way to build the reference row *without* the builder (usually the database parsing JSON). +- `writes`: which writes its builder compiles (`insertSelect`, `update`, `delete`, + `onConflict`, `returning`, `defaultCells` for a multi-row VALUES that leaves a + defaulted column out of some rows), and a `scratch` table: its query `Table`, + the DDL that creates and drops it, and `exec` to run that DDL on the first + target. Set a capability from what the builder does, not what the database + could: the suite still generates an unsupported write and requires the builder + to refuse it. A database whose writes are asynchronous (ClickHouse mutations) + must make them visible to the next read (`mutations_sync=2`). - `targets`: the databases it runs on. Each target states its `Semantics`, the points where databases legitimately disagree (NULL ordering, integer division, what an unmatched LEFT JOIN column holds, LIKE case rules, …). The model reads diff --git a/tests/fuzz/build.ts b/tests/fuzz/build.ts index b52c86c..beb5df2 100644 --- a/tests/fuzz/build.ts +++ b/tests/fuzz/build.ts @@ -8,7 +8,7 @@ import { DateTime, Effect, Exit } from "effect" import type * as CH from "@maple-dev/effect-orm/clickhouse" import type { CoreContext } from "../core-cases" import type { FuzzDialect } from "./dialects/types" -import type { Agg, Cond, Expr, Operand, Plan } from "./plan" +import type { Agg, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" const pad = (n: number, width = 2) => String(n).padStart(width, "0") /** `YYYY-MM-DD hh:mm:ss.sss` in UTC: the string form a DateTime comparison accepts. */ @@ -18,13 +18,13 @@ export const timeString = (ms: number): string => { } /** Every param operand in the plan, in a fixed walk order, with the name it compiles as. */ -const nameParams = (plan: Plan): { readonly names: Map; readonly params: Record } => { +const nameParams = (plan: Plan, prefix: string): { readonly names: Map; readonly params: Record } => { const names = new Map() const params: Record = { orgId: plan.orgId } const operand = (op: Operand) => { if (op.k === "expr") return expr(op.e) if (op.k !== "param") return - const name = `p${names.size}` + const name = `${prefix}p${names.size}` names.set(op, name) params[name] = op.kind === "dateTime" ? DateTime.makeUnsafe(op.v as number) : op.v } @@ -74,9 +74,10 @@ export interface Built { readonly params: Record } -export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Built => { +/** `prefix` keeps the params of union branches apart: each branch names its own from `p0`. */ +export const buildPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan, prefix = ""): Built => { const { lit, param, and, or, not, rawCond, fromQuery } = dialect.module - const { names, params } = nameParams(plan) + const { names, params } = nameParams(plan, prefix) const expr = ($: any, e: Expr): any => { switch (e.k) { @@ -196,10 +197,9 @@ export type Compiled = * `QueryBuilderError` is `Refused`; a die is a `Defect`; an exception that * escapes `compile` — or the builder calls before it — is `Threw`. */ -export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => { +export const classify = (compile: () => Effect.Effect, unknown>): Compiled => { try { - const { query, params } = buildPlan(dialect, ctx, plan) - const exit = Effect.runSyncExit(dialect.module.compile(query, params)) + const exit = Effect.runSyncExit(compile()) if (Exit.isSuccess(exit)) return { _tag: "Compiled", compiled: exit.value } const failure = exit.cause.reasons.find((reason) => reason._tag === "Fail") return failure !== undefined @@ -209,3 +209,26 @@ export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): return { _tag: "Threw", error: error instanceof Error ? `${error.name}: ${error.message}\n${error.stack}` : String(error) } } } + +export const compilePlan = (dialect: FuzzDialect, ctx: CoreContext, plan: Plan): Compiled => + classify(() => { + const { query, params } = buildPlan(dialect, ctx, plan) + return dialect.module.compile(query, params) + }) + +/** A union plan as `unionAll(...)` of its branches, each with its own params. */ +export const buildUnion = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Built => { + const built = plan.branches.map((branch, i) => buildPlan(dialect, ctx, branch, `b${i}`)) + let union: any = (dialect.module.unionAll as (...queries: ReadonlyArray) => unknown)(...built.map((b) => b.query)) + if (plan.orderBy.length > 0) union = union.orderBy(...plan.orderBy.map(([alias, dir]) => [alias, dir])) + if (plan.limit !== undefined) union = union.limit(plan.limit) + if (plan.offset !== undefined) union = union.offset(plan.offset) + if (plan.crossTenant) union = union.crossTenant() + return { query: union, params: Object.assign({}, ...built.map((b) => b.params)) } +} + +export const compileUnionPlan = (dialect: FuzzDialect, ctx: CoreContext, plan: UnionPlan): Compiled => + classify(() => { + const { query, params } = buildUnion(dialect, ctx, plan) + return dialect.module.compileUnion(query, params) + }) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index 0ea4fba..f130294 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -58,12 +58,20 @@ const semantics: Semantics = { likeIgnoresCase: false, } +/** + * An UPDATE is an `ALTER TABLE … UPDATE` mutation, which runs in the background + * unless asked to wait, and a lightweight DELETE may too. Waiting makes a write + * visible to the read that follows it, as the writes suite needs. + */ +const syncWrites = { mutations_sync: "2", lightweight_deletes_sync: "2" } + const target = (name: string, joinUseNulls: boolean): FuzzTarget => ({ name, available: clickhouseEndpoint !== undefined, semantics: joinUseNulls ? { ...semantics, missingJoinColumn: "null" } : semantics, run: (compiled) => runClickHouse(compiled, { + ...syncWrites, join_use_nulls: joinUseNulls ? "1" : "0", output_format_json_quote_64bit_integers: joinUseNulls ? "1" : "0", }), @@ -132,5 +140,26 @@ export const clickhouse: FuzzDialect = { strings, ilike: true, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: false, + returning: false, + defaultCells: true, + scratch: { + table: (name) => + CH.table(name, { + external: true, + columns: { Id: CH.int64, OrgId: CH.string, Name: CH.string, Note: CH.nullable(CH.string), Qty: CH.column(CH.int64, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE ${name} (Id Int64, OrgId String, Name String, Note Nullable(String), Qty Int64 DEFAULT 5) ENGINE = MergeTree ORDER BY Id`], + drop: (name) => `DROP TABLE IF EXISTS ${name}`, + exec: async (sql) => { + const { ok, text } = await clickhouseText(sql, syncWrites) + if (!ok) throw new Error(text.trim()) + }, + }, + }, targets: [target("clickhouse", false), target("clickhouse-join-nulls", true)], } diff --git a/tests/fuzz/dialects/postgres.ts b/tests/fuzz/dialects/postgres.ts index cb10bdc..cd5cb22 100644 --- a/tests/fuzz/dialects/postgres.ts +++ b/tests/fuzz/dialects/postgres.ts @@ -69,6 +69,26 @@ export const postgres: FuzzDialect = { : `placeholders ${[...used].sort((a, b) => a - b).join(",")} do not match ${count} parameters` }, literals, + writes: { + insertSelect: true, + update: true, + delete: true, + onConflict: true, + returning: true, + defaultCells: true, + scratch: { + table: (name) => + PG.table(name, { + external: true, + columns: { Id: PG.int8, OrgId: PG.text, Name: PG.text, Note: PG.nullable(PG.text), Qty: PG.column(PG.int8, { default: 5 }) }, + }), + create: (name) => [`CREATE TABLE "${name}" ("Id" int8 PRIMARY KEY, "OrgId" text NOT NULL, "Name" text NOT NULL, "Note" text, "Qty" int8 NOT NULL DEFAULT 5)`], + drop: (name) => `DROP TABLE IF EXISTS "${name}"`, + exec: async (sql) => { + await database().exec(sql) + }, + }, + }, targets: [ { name: "postgres", diff --git a/tests/fuzz/dialects/types.ts b/tests/fuzz/dialects/types.ts index 3e23e06..51ef8d8 100644 --- a/tests/fuzz/dialects/types.ts +++ b/tests/fuzz/dialects/types.ts @@ -81,6 +81,41 @@ export interface DialectModule { readonly not: typeof CH.not readonly rawCond: typeof CH.rawCond readonly fromQuery: typeof CH.fromQuery + readonly unionAll: typeof CH.unionAll + readonly compileUnion: (union: any, params?: Record) => Effect.Effect, unknown> + readonly insertInto: typeof CH.insertInto + readonly update: typeof CH.update + readonly deleteFrom: typeof CH.deleteFrom +} + +/** + * Which writes the dialect's builder compiles. A write it does not support is + * still generated, and must be refused at compile time rather than run. + */ +export interface WriteSupport { + /** `insertInto(t).select(query)`. */ + readonly insertSelect: boolean + readonly update: boolean + readonly delete: boolean + /** `onConflictDoNothing` / `onConflictDoUpdate`. */ + readonly onConflict: boolean + /** `.returning()` on INSERT, UPDATE and DELETE. */ + readonly returning: boolean + /** A multi-row VALUES where some rows leave out a defaulted column (`DEFAULT` per cell). */ + readonly defaultCells: boolean + /** + * The writes suite's scratch table, on the dialect's first target: a key + * `Id` (integer, unique), `OrgId` and `Name` (text), `Note` (nullable text) + * and `Qty` (integer, database default 5, so an insert may leave it out). + */ + readonly scratch: { + readonly table: (name: string) => CH.Table + /** Statements that create the table, empty. */ + readonly create: (name: string) => ReadonlyArray + readonly drop: (name: string) => string + /** Run one statement on the first target, failing if the database does. */ + readonly exec: (sql: string) => Promise + } } export interface FuzzDialect { @@ -99,7 +134,9 @@ export interface FuzzDialect { readonly checkPlaceholders?: (sql: string, parameterCount: number) => string | undefined /** Column types the literal round-trip property covers. */ readonly literals: ReadonlyArray - /** The databases this dialect's SQL runs on. The first is the literal suite's. */ + /** What the writes suite may generate, and how it gets a table to write to. */ + readonly writes: WriteSupport + /** The databases this dialect's SQL runs on. The first is the literal and writes suites'. */ readonly targets: ReadonlyArray /** Release a connection or process the targets opened. */ readonly close?: () => Promise diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index f170eb8..0bf51b9 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -41,7 +41,8 @@ export const knownHostileIssue = (input: HostileInput): string | undefined => // Query plans (queries.fuzz.test.ts) -import type { Plan } from "./plan" +import { chKind } from "./model" +import type { Plan, UnionPlan } from "./plan" export interface PlanInput { /** The target name (`dialects/*.ts`), or the dialect name for compile-only properties. */ @@ -89,3 +90,39 @@ const planIssues: ReadonlyArray> = [ ] export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) + +// Unions (queries.fuzz.test.ts) + +export interface UnionInput { + readonly target: string + readonly plan: UnionPlan +} + +/** The expression a branch selects under `alias`. */ +const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) + +const unionIssues: ReadonlyArray> = [ + { + id: "clickhouse-union-variant-order-by", + reason: + "On ClickHouse 26, a UNION ALL column whose branches have numeric types with no common supertype (`Id + 1` is " + + "UInt64, `Id - 1` Int64; Int64 and Float64) becomes a Variant, and the union's ORDER BY on it fails with " + + "`Data types Variant/Dynamic are not allowed in ORDER BY keys`. Fix: have compileUnion cast a numeric union " + + "column to one type (e.g. Float64, or the widest signed type) when its branches' expressions differ.", + matches: ({ target, plan }) => + target.startsWith("clickhouse") && + plan.orderBy.some(([alias]) => { + const kinds = new Set( + plan.branches.map((branch) => { + const e = selected(branch, alias) + return e === undefined || (e.k !== "num" && e.k !== "arith" && !(e.k === "col" && (e.col === "Id" || e.col === "Amount"))) + ? "other" + : `${chKind(e)}:${e.k === "col" ? e.col : e.k}` + }), + ) + return kinds.size > 1 && !kinds.has("other") + }), + }, +] + +export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) diff --git a/tests/fuzz/model.ts b/tests/fuzz/model.ts index 68f7889..b04ab13 100644 --- a/tests/fuzz/model.ts +++ b/tests/fuzz/model.ts @@ -12,7 +12,7 @@ import { fixtureRows } from "../core-cases" import type { Semantics } from "./dialects/types" import { skip } from "./harness" -import type { Agg, AggShape, ArithOp, Cond, Expr, Operand, Plan, RowsShape } from "./plan" +import type { Agg, ArithOp, Cond, Expr, Operand, Plan, UnionPlan } from "./plan" type V = number | string | null type Bool = boolean | null @@ -69,7 +69,7 @@ const isTime = (e: Expr) => e.k === "col" && e.col === "Created" * database does (`-7 % toUInt32(3)` is 0), so the model declines those. */ type ChKind = "unsigned" | "signed" | "float" -const chKind = (e: Expr | number): ChKind => { +export const chKind = (e: Expr | number): ChKind => { if (typeof e === "number") return !Number.isInteger(e) ? "float" : e >= 0 ? "unsigned" : "signed" switch (e.k) { case "col": @@ -310,7 +310,7 @@ const sortRows = (env: Env, rows: Array, orderBy: ReadonlyArray(rows: ReadonlyArray, shape: RowsShape | AggShape): ReadonlyArray => { +const page = (rows: ReadonlyArray, shape: { readonly limit?: number | undefined; readonly offset?: number | undefined }): ReadonlyArray => { const start = shape.offset ?? 0 return rows.slice(start, shape.limit === undefined ? undefined : start + shape.limit) } @@ -393,3 +393,18 @@ export const evaluate = (plan: Plan, sem: Semantics): ModelResult => { sortRows(env, rows, shape.orderBy) return { rows: page(rows, shape), ordered: total && !plan.wrap, orgs } } + +/** + * A union: each branch's rows (its own ORDER BY and LIMIT applied first), + * concatenated, then the union's ORDER BY, LIMIT and OFFSET. Rows compare as + * a list only when the union orders by every alias. + */ +export const evaluateUnion = (plan: UnionPlan, sem: Semantics): ModelResult => { + const parts = plan.branches.map((branch) => evaluate(branch, sem)) + const rows = parts.flatMap((part) => [...part.rows]) + const first = plan.branches[0]! + const aliases = first.shape.k === "rows" ? first.shape.select.map(([alias]) => alias) : [] + sortRows({ sem, plan: first, joined: false }, rows, plan.orderBy) + const total = plan.orderBy.length > 0 && aliases.every((alias) => plan.orderBy.some(([key]) => key === alias)) + return { rows: page(rows, plan), ordered: total, orgs: new Set(parts.flatMap((part) => [...part.orgs])) } +} diff --git a/tests/fuzz/plan.ts b/tests/fuzz/plan.ts index 3e9c7a3..e1ba22a 100644 --- a/tests/fuzz/plan.ts +++ b/tests/fuzz/plan.ts @@ -339,17 +339,85 @@ const aggShape = (joined: boolean, options: GenOptions): fc.Arbitrary })) }) +const orgIdArb = fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }) +const joinArb = fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }) + +// Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. +const whereArb = (joined: boolean, options: GenOptions): fc.Arbitrary> => + fc.oneof( + fc.constant([]), + fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), + ) + +/** + * `unionAll(branch, …)` with the union's own ORDER BY, LIMIT and OFFSET. Each + * branch is a row plan selecting the same aliases, each alias of one sort in + * every branch, so the union is well-typed by construction; a branch may have + * its own join, filter, DISTINCT, ORDER BY and LIMIT. + */ +export interface UnionPlan { + readonly orgId: string + readonly branches: ReadonlyArray + readonly orderBy: ReadonlyArray + readonly limit?: number + readonly offset?: number + readonly crossTenant: boolean +} + +const branch = (orgId: string, sorts: ReadonlyArray, options: GenOptions): fc.Arbitrary => + joinArb.chain((join) => { + const joined = join !== undefined + return fc.tuple(...sorts.map((sort) => expr(sort, joined, 2, options))).chain((exprs) => { + const select: Array = [["id", { k: "col", col: "Id" }], ...exprs.map((e, i) => [`e${i}`, e] as const)] + return fc.record({ where: whereArb(joined, options), distinct: fc.boolean(), order: ordering(select.map(([a]) => a), ["id"]) }).map( + ({ where, distinct, order }): Plan => ({ orgId, join, where, shape: { k: "rows", select, distinct, ...order }, wrap: false, crossTenant: false }), + ) + }) + }) + +/** + * The union's ordering. Rows of different branches can tie on every alias but + * one, so it pages only when ordered by every alias: then tied rows are equal + * rows, and which of them LIMIT keeps does not matter. + */ +const unionOrdering = (aliases: ReadonlyArray) => + fc.oneof( + fc.constant({ orderBy: [] as ReadonlyArray, limit: undefined, offset: undefined }), + fc + .record({ + keys: fc.shuffledSubarray([...aliases], { minLength: aliases.length }), + dirs: fc.array(fc.constantFrom("asc" as const, "desc" as const), { minLength: aliases.length, maxLength: aliases.length }), + limit: fc.option(fc.integer({ min: 0, max: 8 }), { nil: undefined }), + offset: fc.option(fc.integer({ min: 0, max: 4 }), { nil: undefined }), + }) + .map(({ keys, dirs, limit, offset }) => ({ orderBy: keys.map((k, i) => [k, dirs[i]!] as const), limit, offset })), + fc.shuffledSubarray([...aliases], { minLength: 1 }).map((keys) => ({ orderBy: keys.map((k) => [k, "asc" as const] as const), limit: undefined, offset: undefined })), + ) + +export const unionPlan = (options: GenOptions): fc.Arbitrary => + fc + .record({ + orgId: orgIdArb, + sorts: fc.array(fc.constantFrom("num", "str", "nstr", "time"), { maxLength: 3 }), + branches: fc.integer({ min: 2, max: 3 }), + crossTenant: fc.boolean(), + }) + .chain(({ orgId, sorts, branches, crossTenant }) => + fc + .record({ + branches: fc.tuple(...Array.from({ length: branches }, () => branch(orgId, sorts, options))), + order: unionOrdering(["id", ...sorts.map((_, i) => `e${i}`)]), + }) + .map(({ branches, order }): UnionPlan => ({ orgId, branches, crossTenant, ...order })), + ) + export const plan = (options: GenOptions): fc.Arbitrary => - fc.option(fc.constantFrom("inner" as const, "left" as const), { nil: undefined, freq: 3 }).chain((join) => { + joinArb.chain((join) => { const joined = join !== undefined return fc.record({ - orgId: fc.oneof({ weight: 6, arbitrary: fc.constant("org_1") }, { weight: 2, arbitrary: fc.constant("org_2") }, { weight: 1, arbitrary: fc.constant("org_3") }), + orgId: orgIdArb, join: fc.constant(join), - // Short: every entry is ANDed, and a long conjunction of random predicates is almost always empty. - where: fc.oneof( - fc.constant([]), - fc.array(fc.oneof({ weight: 1, arbitrary: fc.constant({ k: "tenant", side: "orders" }) }, { weight: 3, arbitrary: cond(joined, 2, options) }), { minLength: 1, maxLength: 2 }), - ), + where: whereArb(joined, options), shape: fc.oneof({ weight: 3, arbitrary: rowsShape(joined, options) }, { weight: 2, arbitrary: aggShape(joined, options) }), wrap: fc.constantFrom(false as const, "all" as const, "names" as const), crossTenant: fc.boolean(), diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index 49730f1..c836dd8 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -5,15 +5,18 @@ // // tenant soundness a query that reports "single-tenant" read rows of one org only // +// Unions are fuzzed the same way: `unionAll` of row plans, each branch with its +// own ORDER BY and LIMIT, under the union's own ORDER BY, LIMIT and OFFSET. +// // Every target in the registry (dialects/index.ts) runs; one whose database is // not reachable here is skipped, not dropped, so the run says so. import { afterAll, describe, expect, it } from "vitest" -import { compilePlan, scopeContext } from "./build" +import { compilePlan, compileUnionPlan, scopeContext, type Compiled } from "./build" import { closeAll, targets, type FuzzDialect, type FuzzTarget } from "./dialects" import { FuzzFailure, property, skipKnown, withContext } from "./harness" -import { knownPlanIssue } from "./known-issues" -import { evaluate } from "./model" -import { nastyOptions, plainOptions, plan, type Plan } from "./plan" +import { knownPlanIssue, knownUnionIssue } from "./known-issues" +import { evaluate, evaluateUnion, type ModelResult } from "./model" +import { nastyOptions, plainOptions, plan, unionPlan, type Plan, type UnionPlan } from "./plan" import { normalize } from "./values" afterAll(closeAll) @@ -27,15 +30,33 @@ const canonical = (rows: ReadonlyArray): Array => ) const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort() -const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { +const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - const expected = evaluate(input, target.semantics) - const result = compilePlan(dialect, dialect.context, input) + return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input)) +} + +const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { + const knownUnion = knownUnionIssue({ target: target.name, plan: input }) + if (knownUnion !== undefined) skipKnown(knownUnion) + for (const branch of input.branches) { + const known = knownPlanIssue({ target: target.name, plan: branch }) + if (known !== undefined) skipKnown(known) + } + return checkRows( + target, + input.orgId, + evaluateUnion(input, target.semantics), + compileUnionPlan(dialect, dialect.context, input), + compileUnionPlan(dialect, scopeContext(dialect), input), + ) +} + +/** Run what compiled and compare its rows with the model's; `scoped` is the same input over the plain tables. */ +const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled) => { if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. - const scoped = compilePlan(dialect, scopeContext(dialect), input) const tenantScope = scoped._tag === "Compiled" ? scoped.compiled.tenantScope : undefined const outcome = await target.run(compiled) await withContext({ sql: compiled.sql, parameters: compiled.parameters, tenantScope, outcome, expected: expected.rows }, () => { @@ -46,7 +67,7 @@ const matchesModel = async (dialect: FuzzDialect, target: FuzzTarget, input: Pla if (expected.ordered) expect(actual).toEqual(want) else expect(sorted(actual)).toEqual(sorted(want)) if (tenantScope === "single-tenant") { - const foreign = [...expected.orgs].filter((org) => org !== input.orgId) + const foreign = [...expected.orgs].filter((org) => org !== orgId) expect(foreign, "a single-tenant query read another org's rows").toEqual([]) } }) @@ -63,3 +84,15 @@ describe("queries match the model", () => { } } }) + +describe("unions match the model", () => { + for (const { dialect, target } of targets) { + for (const [label, options] of [ + ["plain strings", plainOptions(dialect)], + ["nasty strings", nastyOptions(dialect)], + ] as const) { + const name = `unions match the model (${target.name}, ${label})` + it.skipIf(!target.available)(name, () => property(name, unionPlan(options), (input) => unionMatchesModel(dialect, target, input), { runs: 60 }), 300_000) + } + } +}) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index a21260e..283f51a 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -5,7 +5,7 @@ // target at once). Inputs that hit an open entry in `known-issues.ts` are // skipped until the bug is fixed, then replayed for good. import { DateTime } from "effect" -import type { Plan, RowsShape } from "./plan" +import type { Plan, RowsShape, UnionPlan } from "./plan" const rows = (shape: RowsShape): Plan => ({ orgId: "org_1", join: undefined, where: [], shape, wrap: false, crossTenant: false }) @@ -51,6 +51,20 @@ export const pinned: Record> = { "postgres:text[] select": [[]], "postgres:timestamptz": [DateTime.makeUnsafe(Date.UTC(100, 0, 1))], + // Unions. + "unions match the model": [ + // clickhouse-union-variant-order-by: `Id + 1` (UInt64) under `Id + -1` (Int64), ordered by that column. + { + orgId: "org_1", + branches: [ + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: 1 }]], distinct: false, orderBy: [] }), + rows({ k: "rows", select: [["id", { k: "col", col: "Id" }], ["e0", { k: "arith", op: "add", l: { k: "col", col: "Id" }, r: -1 }]], distinct: false, orderBy: [] }), + ], + orderBy: [["e0", "asc"], ["id", "asc"]], + crossTenant: false, + }, + ] satisfies ReadonlyArray, + // Hostile values. "hostile values are refused, never thrown (clickhouse)": [ { dialect: "clickhouse", column: "OrgId", site: "eq", value: 0n }, diff --git a/tests/fuzz/writes.fuzz.test.ts b/tests/fuzz/writes.fuzz.test.ts new file mode 100644 index 0000000..49ff3ac --- /dev/null +++ b/tests/fuzz/writes.fuzz.test.ts @@ -0,0 +1,351 @@ +// Writes: random sequences of builder writes leave the table the model says. +// +// Each input gets a fresh scratch table on the dialect's first target (see +// `WriteSupport.scratch`), then runs a sequence of writes through the builder: +// multi-row inserts that leave defaulted or nullable columns out, INSERT … +// SELECT, ON CONFLICT DO NOTHING / DO UPDATE (with the existing and the +// proposed row), UPDATE and DELETE with a filter or `allRows()`, and RETURNING. +// After every write the table is read back and compared with a JS model of its +// rows, and RETURNING rows with the rows the model says the write touched. +// +// A write the dialect does not support (`WriteSupport`) is still generated, and +// must be refused at compile time: never run, never thrown. +import fc from "fast-check" +import { afterAll, describe, expect, it } from "vitest" +import { classify } from "./build" +import { closeAll, dialects, type FuzzDialect } from "./dialects" +import type { WriteSupport } from "./dialects/types" +import { FuzzFailure, property, skip, withContext } from "./harness" +import { nastyString, wellFormed } from "./values" + +afterAll(closeAll) + +// Plans + +interface Row { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note: string | null + readonly Qty: number +} +/** A row to insert; `Note` and `Qty` may be left out (NULL, and the database default 5). */ +interface NewRow { + readonly Id: number + readonly OrgId: string + readonly Name: string + readonly Note?: string | null + readonly Qty?: number +} +type QtySet = { readonly k: "value"; readonly v: number } | { readonly k: "add"; readonly v: number } | { readonly k: "excluded" } +interface SetPlan { + readonly Name?: string + readonly Note?: string | null + readonly Qty?: QtySet +} +type Where = + | { readonly k: "id"; readonly op: "eq" | "lt" | "gt"; readonly v: number } + | { readonly k: "name"; readonly v: string } + | { readonly k: "note"; readonly isNull: boolean } + | { readonly k: "org"; readonly v: string } + | { readonly k: "and" | "or"; readonly a: Where; readonly b: Where } +type Conflict = { readonly action: "nothing"; readonly target: boolean } | { readonly action: "update"; readonly set: SetPlan } +type WriteOp = + | { readonly k: "insert"; readonly rows: ReadonlyArray; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "insertSelect"; readonly offset: number; readonly where: Where; readonly conflict?: Conflict; readonly returning: boolean } + | { readonly k: "update"; readonly set: SetPlan; readonly where: Where | "all"; readonly returning: boolean } + | { readonly k: "delete"; readonly where: Where | "all"; readonly returning: boolean } +export interface WritesInput { + readonly ops: ReadonlyArray +} + +const DEFAULT_QTY = 5 +const ids = fc.integer({ min: 1, max: 12 }) +const org = fc.constantFrom("org_1", "org_2") + +const generators = (dialect: FuzzDialect) => { + const text = fc.oneof( + { weight: 3, arbitrary: fc.constantFrom("a", "b", "c", "it's", "x\\y") }, + { weight: 1, arbitrary: nastyString(dialect.strings).filter(wellFormed) }, + ) + const newRow: fc.Arbitrary = fc.record( + { Id: ids, OrgId: org, Name: text, Note: fc.option(text, { nil: null }), Qty: fc.integer({ min: -5, max: 50 }) }, + { requiredKeys: ["Id", "OrgId", "Name"] }, + ) + const qty = (excluded: boolean): fc.Arbitrary => + fc.oneof( + fc.integer({ min: -5, max: 50 }).map((v): QtySet => ({ k: "value", v })), + fc.integer({ min: -3, max: 3 }).map((v): QtySet => ({ k: "add", v })), + ...(excluded ? [fc.constant({ k: "excluded" })] : []), + ) + const set = (excluded: boolean): fc.Arbitrary => + fc + .record({ Name: text, Note: fc.option(text, { nil: null }), Qty: qty(excluded) }, { requiredKeys: [] }) + .filter((s) => Object.keys(s).length > 0) + const leaf: fc.Arbitrary = fc.oneof( + fc.record({ k: fc.constant("id" as const), op: fc.constantFrom("eq" as const, "lt" as const, "gt" as const), v: ids }), + fc.record({ k: fc.constant("name" as const), v: text }), + fc.record({ k: fc.constant("note" as const), isNull: fc.boolean() }), + fc.record({ k: fc.constant("org" as const), v: org }), + ) + const where: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: leaf }, + { weight: 1, arbitrary: fc.record({ k: fc.constantFrom("and" as const, "or" as const), a: leaf, b: leaf }) }, + ) + const conflict: fc.Arbitrary = fc.option( + fc.oneof( + fc.record({ action: fc.constant("nothing" as const), target: fc.boolean() }), + fc.record({ action: fc.constant("update" as const), set: set(true) }), + ), + { nil: undefined, freq: 2 }, + ) + const returning = fc.boolean() + const op: fc.Arbitrary = fc.oneof( + { weight: 3, arbitrary: fc.record({ k: fc.constant("insert" as const), rows: fc.array(newRow, { minLength: 1, maxLength: 4 }), conflict, returning }, { requiredKeys: ["k", "rows", "returning"] }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("insertSelect" as const), offset: fc.constantFrom(100, 1000, 10_000), where, conflict, returning }, { requiredKeys: ["k", "offset", "where", "returning"] }) }, + { weight: 2, arbitrary: fc.record({ k: fc.constant("update" as const), set: set(false), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + { weight: 1, arbitrary: fc.record({ k: fc.constant("delete" as const), where: fc.oneof({ weight: 4, arbitrary: where }, { weight: 1, arbitrary: fc.constant("all" as const) }), returning }) }, + ) + // The first write seeds rows, so later ones have something to change. + const seed = fc.array(newRow, { minLength: 1, maxLength: 5 }).map((rows): WriteOp => ({ k: "insert", rows, returning: false })) + return fc.tuple(seed, fc.array(op, { minLength: 1, maxLength: 5 })).map(([first, rest]): WritesInput => ({ ops: [first, ...rest] })) +} + +// The model + +const matches = (where: Where | "all", row: Row): boolean => { + if (where === "all") return true + switch (where.k) { + case "id": + return where.op === "eq" ? row.Id === where.v : where.op === "lt" ? row.Id < where.v : row.Id > where.v + case "name": + return row.Name === where.v + case "note": + return (row.Note === null) === where.isNull + case "org": + return row.OrgId === where.v + case "and": + return matches(where.a, row) && matches(where.b, row) + case "or": + return matches(where.a, row) || matches(where.b, row) + } +} + +const applySet = (set: SetPlan, row: Row, excluded?: Row): Row => ({ + ...row, + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined + ? {} + : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? row.Qty + set.Qty.v : excluded!.Qty }), +}) + +const fullRow = (row: NewRow): Row => ({ Id: row.Id, OrgId: row.OrgId, Name: row.Name, Note: row.Note ?? null, Qty: row.Qty ?? DEFAULT_QTY }) + +/** + * Rows inserted one by one, as the database does: a key that exists is a + * conflict, skipped or updated. Returns the rows the statement inserted or + * updated, which is what RETURNING gives. + */ +const insertRows = (state: Array, rows: ReadonlyArray, conflict: Conflict | undefined): Array => { + const touched: Array = [] + for (const row of rows) { + const at = state.findIndex((r) => r.Id === row.Id) + if (at === -1) { + state.push(row) + touched.push(row) + } else if (conflict?.action === "update") { + state[at] = applySet(conflict.set, state[at]!, row) + touched.push(state[at]!) + } + } + return touched +} + +/** Apply a write to the model; the rows RETURNING gives. */ +const apply = (state: Array, op: WriteOp): Array => { + switch (op.k) { + case "insert": + return insertRows(state, op.rows.map(fullRow), op.conflict) + case "insertSelect": + // The SELECT reads the table as it was before the statement. + return insertRows(state, state.filter((row) => matches(op.where, row)).map((row) => ({ ...row, Id: row.Id + op.offset })), op.conflict) + case "update": { + const touched: Array = [] + for (const [i, row] of state.entries()) { + if (!matches(op.where, row)) continue + state[i] = applySet(op.set, row) + touched.push(state[i]!) + } + return touched + } + case "delete": { + const removed = state.filter((row) => matches(op.where, row)) + const kept = state.filter((row) => !matches(op.where, row)) + state.splice(0, state.length, ...kept) + return removed + } + } +} + +/** + * Make a write valid for the table it meets: an insert without ON CONFLICT + * gets fresh keys where its keys exist (a duplicate key is an error, not a + * finding), and one with DO UPDATE no repeated key (Postgres refuses to update + * a row twice in one statement). + */ +const repair = (state: ReadonlyArray, used: Set, op: WriteOp): WriteOp => { + const fresh = () => { + let id = Math.max(0, ...used) + 1 + while (used.has(id)) id++ + return id + } + if (op.k === "insert") { + const seen = new Set() + const rows = op.rows.map((row) => { + const clash = op.conflict === undefined ? state.some((r) => r.Id === row.Id) || seen.has(row.Id) : op.conflict.action === "update" && seen.has(row.Id) + const id = clash ? fresh() : row.Id + seen.add(id) + used.add(id) + return id === row.Id ? row : { ...row, Id: id } + }) + return { ...op, rows } + } + if (op.k === "insertSelect" && op.conflict === undefined) { + const copies = state.filter((row) => matches(op.where, row)).map((row) => row.Id + op.offset) + if (copies.some((id) => state.some((r) => r.Id === id))) skip("INSERT … SELECT would repeat a key") + for (const id of copies) used.add(id) + } + if (op.k === "insertSelect") for (const row of state) used.add(row.Id + op.offset) + return op +} + +// Capabilities + +/** The first feature `op` uses that the dialect does not support, if any. */ +const unsupported = (support: WriteSupport, op: WriteOp): string | undefined => { + if (op.k === "insertSelect" && !support.insertSelect) return "INSERT … SELECT" + if (op.k === "update" && !support.update) return "UPDATE" + if (op.k === "delete" && !support.delete) return "DELETE" + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined && !support.onConflict) return "ON CONFLICT" + if (op.returning && !support.returning) return "RETURNING" + if (op.k === "insert" && !support.defaultCells && op.rows.length > 1) { + for (const column of ["Note", "Qty"] as const) { + const given = op.rows.filter((row) => Object.hasOwn(row, column)).length + if (given > 0 && given < op.rows.length) return `DEFAULT for a missing ${column} in a multi-row VALUES` + } + } + return undefined +} + +// Builder calls + +const whereCond = ($: any, where: Where): any => { + switch (where.k) { + case "id": + return $.Id[where.op](where.v) + case "name": + return $.Name.eq(where.v) + case "note": + return where.isNull ? $.Note.isNull() : $.Note.isNotNull() + case "org": + return $.OrgId.eq(where.v) + case "and": + case "or": + return whereCond($, where.a)[where.k](whereCond($, where.b)) + } +} + +const setRecord = (set: SetPlan, $: any, excluded?: any): Record => ({ + ...(set.Name !== undefined ? { Name: set.Name } : {}), + ...(set.Note !== undefined ? { Note: set.Note } : {}), + ...(set.Qty === undefined ? {} : { Qty: set.Qty.k === "value" ? set.Qty.v : set.Qty.k === "add" ? $.Qty.add(set.Qty.v) : excluded.Qty }), +}) + +const buildWrite = (dialect: FuzzDialect, table: any, op: WriteOp): any => { + const { insertInto, update, deleteFrom, from } = dialect.module + let write: any + const filtered = (w: any, where: Where | "all") => (where === "all" ? w.allRows() : w.where(($: any) => [whereCond($, where)])) + switch (op.k) { + case "insert": + write = insertInto(table).values(op.rows.length === 1 ? (op.rows[0] as never) : (op.rows as never)) + break + case "insertSelect": + write = insertInto(table).select( + (from(table) as any) + .select(($: any) => ({ Id: $.Id.add(op.offset), OrgId: $.OrgId, Name: $.Name, Note: $.Note, Qty: $.Qty })) + .where(($: any) => [whereCond($, op.where)]), + ) + break + case "update": + write = filtered(update(table).set(($: any) => setRecord(op.set, $)), op.where) + break + case "delete": + write = filtered(deleteFrom(table), op.where) + break + } + if ((op.k === "insert" || op.k === "insertSelect") && op.conflict !== undefined) { + const conflict = op.conflict + write = + conflict.action === "nothing" + ? write.onConflictDoNothing(conflict.target ? { target: ["Id"] } : {}) + : write.onConflictDoUpdate({ target: ["Id"], set: ($: any, excluded: any) => setRecord(conflict.set, $, excluded) }) + } + return op.returning ? write.returning() : write +} + +// The property + +const byId = (rows: ReadonlyArray) => [...rows].sort((a, b) => a.Id - b.Id) +const asRows = (rows: ReadonlyArray): Array => + rows.map((row) => { + const r = row as Record + return { Id: Number(r.Id), OrgId: r.OrgId as string, Name: r.Name as string, Note: (r.Note ?? null) as string | null, Qty: Number(r.Qty) } + }) + +let tables = 0 + +const writesMatchModel = async (dialect: FuzzDialect, input: WritesInput) => { + const { scratch } = dialect.writes + const target = dialect.targets[0]! + const name = `fuzz_writes_${process.pid}_${tables++}` + for (const sql of scratch.create(name)) await scratch.exec(sql) + try { + const table = scratch.table(name) + const read = classify(() => dialect.module.compile((dialect.module.from(table) as any).select(), {})) + if (read._tag !== "Compiled") throw new FuzzFailure("the scratch table's SELECT did not compile", { error: read.error }) + const state: Array = [] + const used = new Set() + for (const [step, planned] of input.ops.entries()) { + const op = repair(state, used, planned) + const missing = unsupported(dialect.writes, op) + const result = classify(() => dialect.module.compile(buildWrite(dialect, table, op), {})) + const context = { step, op, compiled: result._tag === "Compiled" ? { sql: result.compiled.sql, parameters: result.compiled.parameters } : result, before: byId(state) } + if (result._tag === "Threw") throw new FuzzFailure("the builder threw instead of failing", context) + if (missing !== undefined) { + if (result._tag === "Compiled") throw new FuzzFailure(`${missing} compiled for ${dialect.name}, which does not support it`, context) + continue + } + if (result._tag !== "Compiled") throw new FuzzFailure(`a supported write was not compiled (${result._tag})`, context) + const touched = apply(state, op) + const outcome = await target.run(result.compiled) + const after = await target.run(read.compiled) + await withContext({ ...context, outcome, expected: byId(state), returningExpected: op.returning ? byId(touched) : undefined, after }, () => { + expect(outcome._tag).toBe("Rows") + expect(after._tag).toBe("Rows") + if (outcome._tag !== "Rows" || after._tag !== "Rows") return + expect(byId(asRows(after.rows))).toEqual(byId(state)) + if (op.returning) expect(byId(asRows(outcome.rows))).toEqual(byId(touched)) + }) + } + } finally { + await scratch.exec(scratch.drop(name)) + } +} + +describe("writes match the model", () => { + for (const dialect of dialects) { + const name = `writes match the model (${dialect.name})` + it.skipIf(!dialect.targets[0]!.available)(name, () => property(name, generators(dialect), (input) => writesMatchModel(dialect, input), { runs: 40 }), 300_000) + } +}) From 7fcd3cab583656478b98b81b0afd6553fdf77d6a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:09:40 +0100 Subject: [PATCH 14/24] fix(clickhouse): cast a union's mixed numeric column to Float64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A UNION ALL column of numbers whose branches have no common ClickHouse type (`Id + 1` is UInt64 and `Id - 1` Int64; Int64 and Float64) becomes a Variant, and an ORDER BY on it fails ("Variant/Dynamic are not allowed in ORDER BY keys"); with use_variant_as_common_type = 0 the union itself is refused. The union fuzzer found it. On a dialect with `unionNumberCast` (ClickHouse), a column is selected as toFloat64(…) in every branch when every branch selects a number and their types may differ: different declared types, or different computations. Every numeric ClickHouse type decodes to a JS number, so Float64 loses nothing the decode keeps. The known-issues entry is gone; the pinned counterexample now passes. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 8 +++++ docs/unions-and-ctes.md | 10 ++++-- src/ch/compile.test.ts | 20 ++++++++++++ src/ch/compile.ts | 45 +++++++++++++++++++++++--- src/ch/dialect.ts | 9 ++++++ tests/fuzz/known-issues.ts | 22 +------------ 6 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 .changeset/clickhouse-union-numbers.md diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md new file mode 100644 index 0000000..e92afff --- /dev/null +++ b/.changeset/clickhouse-union-numbers.md @@ -0,0 +1,8 @@ +--- +"@maple-dev/effect-orm": patch +--- + +On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and +`Id - 1`, or a `UInt32` and an `Int64` column) is selected as `toFloat64(…)` in each branch. +ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic +are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 8c3f5e2..da64a04 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -28,9 +28,13 @@ themselves keep whatever ordering they declared. Unions compile with **`compileUnion`**, not `compile`. -> **Watch branch types.** ClickHouse unifies column types across `UNION ALL` branches, and a -> branch pairing (say) a `Float64` with a `UInt64` can be rejected outright depending on server -> settings. Cast explicitly in each branch when the types are not already identical. +> **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and +> `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, +> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose +> branches may differ in type (different declared types, or different arithmetic: `Id + 1` is +> `UInt64`, `Id - 1` is `Int64`) is selected as `toFloat64(…)` in every branch. Numbers decode to +> JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical +> expressions are left as they are. Cast other types explicitly when the branches differ. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index e889029..90dd204 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -159,6 +159,26 @@ describe("CompiledQuery.decodeRows", () => { expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] }) }) + it("casts a ClickHouse union's numeric column to one type when its branches may differ", () => { + // `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses. + const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) + const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) + const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) + expect(compiled.sql).toContain("toFloat64(orders.Id + 1) AS shifted") + expect(compiled.sql).toContain("toFloat64(orders.Id + -1) AS shifted") + // One declared type in every branch, and non-numbers, stay as they are. + expect(compiled.sql).not.toContain("toFloat64(orders.Id) AS id") + expect(compiled.sql).not.toContain("toFloat64(orders.Amount)") + expect(compiled.sql).not.toContain("toFloat64(orders.Name)") + // Different declared types do differ. + const mixed = compileUnionUnsafe( + CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), + {}, + ) + expect(mixed.sql).toContain("toFloat64(orders.Id) AS n") + expect(mixed.sql).toContain("toFloat64(orders.Amount) AS n") + }) + it.effect("decodes rows with the declared schema for handwritten SQL", () => Effect.gen(function* () { const compiled = rawCompiledQuery<{ readonly name: string; readonly count: number }>({ diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..8213f78 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -8,7 +8,7 @@ // 4. Assembling into SqlQuery and calling the existing compileQuery() import { compiled, compiledUnsafe, fail } from "./failure" -import { columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" +import { CHFloatResult, CHNumber, columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" import type { CHQuery, CHQueryState, NeedsSelect } from "./query" import type { CHUnionQuery } from "./union" import { isInsert, type AnyInsert, type CHInsert } from "./insert" @@ -22,7 +22,7 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { track, untracked, type RenderTrack } from "../sql/render-tracker" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param" -import { mergeResultSchemas } from "./define-fn" +import { mergeResultSchemas, schemaOf } from "./define-fn" import { encodeValue } from "./literal" import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" import { Effect, Option, Schema } from "effect" @@ -718,6 +718,8 @@ function compileInner< params: Params, options?: { selectKeys?: ReadonlyArray + /** Union columns cast with the dialect's `unionNumberCast`; see `unionNumberCasts`. */ + castNumbers?: ReadonlySet skipFormat?: boolean rowSchema?: CompiledQueryRowSchema /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. @@ -922,7 +924,8 @@ function compileInner< } const fragment = expr.toFragment() const [sql, found] = track(() => compileSqlFragment(fragment)) - return { alias, sql: castSelectedParam(expr, sql), found, column: plainColumn(fragment) } + const numberCast = options?.castNumbers?.has(alias) === true ? currentDialect().unionNumberCast : undefined + return { alias, sql: numberCast !== undefined ? numberCast(castSelectedParam(expr, sql)) : castSelectedParam(expr, sql), found, column: plainColumn(fragment) } }) const where = presentWhere.map((c) => raw(rowFilter("WHERE", c))) const having = (state.havingFn ? state.havingFn($) : []) @@ -1391,6 +1394,39 @@ export function compileUnionRaw>( } /** The recursion behind {@link compileUnionUnsafe}; see {@link compileInner}. */ +/** A JS-number schema: every numeric ClickHouse type and arithmetic result decodes through one of these. */ +const isNumberSchema = (schema: unknown): boolean => { + if (schema === CHNumber || schema === CHFloatResult) return true + // NullOr(CHNumber): a nullable column or arithmetic over one. + const members = (schema as { members?: ReadonlyArray } | undefined)?.members + return Array.isArray(members) && members.some((m) => m === CHNumber || m === CHFloatResult) && members.every((m) => m === CHNumber || m === CHFloatResult || m === Schema.Null) +} + +/** + * The union columns to cast to one numeric type, on a dialect with a + * `unionNumberCast`: every branch selects a number there, and their types may + * differ, because the branches declare different column types, or compute it + * differently (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). + */ +function unionNumberCasts(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { + const casts = new Set() + if (currentDialect().unionNumberCast === undefined || queries.length < 2) return casts + for (const alias of selectKeys) { + const exprs = queries.map((q) => (selectExprsOf(q) as Record | undefined)?.[alias]) + const kinds = exprs.map((expr) => { + if (!isExprLike(expr)) return undefined + const declared = columnTypeOf(expr as Expr) + if (declared !== undefined) return isNumberSchema(declared.schema) ? declared.sql : undefined + return isNumberSchema(schemaOf(expr)) ? "computed" : undefined + }) + if (kinds.some((kind) => kind === undefined)) continue + if (new Set(kinds).size > 1) casts.add(alias) + // The same computation in every branch (a `sum` over one column) has one type; different ones may not. + else if (kinds[0] === "computed" && new Set(exprs.map((expr) => compileSqlFragment((expr as Expr).toFragment()))).size > 1) casts.add(alias) + } + return casts +} + function compileUnionInner, Params extends Record>( union: CHUnionQuery, params: Params, @@ -1416,8 +1452,9 @@ function compileUnionInner, Params extends Re if (state.queries.some((q) => q._state.lock !== undefined)) { return fail(new QueryBuilderDefect({ message: "unionAll: a branch cannot take a row lock; lock in a query over the union instead" }),failedQuery()) } + const castNumbers = unionNumberCasts(state.queries, selectKeys) const subQueries = state.queries.map((q) => - compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), + compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes, castNumbers }), ) const bounds = new Set( subQueries.flatMap((q) => { diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b6c97..d77dee7 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -151,6 +151,13 @@ export interface Dialect extends Omit { * unchecked: a custom dialect says which set it renders, if either. */ readonly functions?: string + /** + * Casts a numeric UNION ALL column whose branches may have different types + * to one type. ClickHouse has no common type for `UInt64` and `Int64` (or + * `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY + * refuses. Absent: the dialect unifies numeric branches itself. + */ + readonly unionNumberCast?: (sql: string) => string } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -173,6 +180,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", + // Every numeric ClickHouse type decodes to a JS number (CHNumber), so Float64 loses nothing the decode keeps. + unionNumberCast: (sql) => `toFloat64(${sql})`, } // The dialect of the enclosing compile, beside the syntax installed for the diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index 0bf51b9..e487af7 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -102,27 +102,7 @@ export interface UnionInput { const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) const unionIssues: ReadonlyArray> = [ - { - id: "clickhouse-union-variant-order-by", - reason: - "On ClickHouse 26, a UNION ALL column whose branches have numeric types with no common supertype (`Id + 1` is " + - "UInt64, `Id - 1` Int64; Int64 and Float64) becomes a Variant, and the union's ORDER BY on it fails with " + - "`Data types Variant/Dynamic are not allowed in ORDER BY keys`. Fix: have compileUnion cast a numeric union " + - "column to one type (e.g. Float64, or the widest signed type) when its branches' expressions differ.", - matches: ({ target, plan }) => - target.startsWith("clickhouse") && - plan.orderBy.some(([alias]) => { - const kinds = new Set( - plan.branches.map((branch) => { - const e = selected(branch, alias) - return e === undefined || (e.k !== "num" && e.k !== "arith" && !(e.k === "col" && (e.col === "Id" || e.col === "Amount"))) - ? "other" - : `${chKind(e)}:${e.k === "col" ? e.col : e.k}` - }), - ) - return kinds.size > 1 && !kinds.has("other") - }), - }, + ] export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) From f8b057a58055c03fc95b21683ec34eb0f8b724b3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:13:15 +0100 Subject: [PATCH 15/24] test(fuzz): fuzz SQLite and D1 writes and unions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers the write capabilities from #122 for sqlite and d1: every write the dialect has (INSERT … SELECT, UPDATE, DELETE, ON CONFLICT, RETURNING), and per-cell DEFAULT refused, as SQLite has none in VALUES. D1 compiles its unions with the D1 dialect. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/dialects/sqlite.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/fuzz/dialects/sqlite.ts b/tests/fuzz/dialects/sqlite.ts index 84bad67..459aa62 100644 --- a/tests/fuzz/dialects/sqlite.ts +++ b/tests/fuzz/dialects/sqlite.ts @@ -7,7 +7,7 @@ import * as SQLite from "@maple-dev/effect-orm/sqlite" import { d1Context, sqliteContext } from "../../core-cases" import { d1Database, sqliteDatabase } from "../../sqlite-support" import { finite, nastyString, nullOr, utc, type LiteralType } from "../values" -import { runWith, type DialectModule, type FuzzDialect, type Semantics, type StringRules } from "./types" +import { runWith, type DialectModule, type FuzzDialect, type Semantics, type StringRules, type WriteSupport } from "./types" const semantics: Semantics = { missingJoinColumn: "null", @@ -100,6 +100,25 @@ const literals = (dialect: string): ReadonlyArray => { const sqliteDb = sqliteDatabase() const d1Db = d1Database() +/** Everything but per-cell DEFAULT, which the dialect refuses (SQLite has none in VALUES). */ +const writes = (exec: (sql: string) => Promise): WriteSupport => ({ + insertSelect: true, + update: true, + delete: true, + onConflict: true, + returning: true, + defaultCells: false, + scratch: { + table: (name) => + SQLite.table(name, { Id: SQLite.integer, OrgId: SQLite.text, Name: SQLite.text, Note: SQLite.nullable(SQLite.text), Qty: SQLite.integer }, { defaults: ["Qty"] }), + create: (name) => [`CREATE TABLE "${name}" ("Id" integer PRIMARY KEY, "OrgId" text NOT NULL, "Name" text NOT NULL, "Note" text, "Qty" integer NOT NULL DEFAULT 5)`], + drop: (name) => `DROP TABLE IF EXISTS "${name}"`, + exec: async (sql) => { + await exec(sql) + }, + }, +}) + export const sqlite: FuzzDialect = { name: "sqlite", module: SQLite as never, @@ -107,6 +126,7 @@ export const sqlite: FuzzDialect = { strings, ilike: false, literals: literals("sqlite"), + writes: writes((sql) => sqliteDb.query(sql, [])), targets: [{ name: "sqlite", available: true, semantics, run: (compiled) => runWith(compiled, sqliteDb.query) }], close: sqliteDb.close, } @@ -114,6 +134,7 @@ export const sqlite: FuzzDialect = { const d1Module: DialectModule = { ...(SQLite as never as DialectModule), compile: (query, params) => SQLite.compile(query, params, { dialect: SQLite.d1Dialect }), + compileUnion: (union, params) => SQLite.compileUnion(union, params, { dialect: SQLite.d1Dialect }), } export const d1: FuzzDialect = { @@ -123,6 +144,7 @@ export const d1: FuzzDialect = { strings, ilike: false, literals: literals("d1"), + writes: writes((sql) => d1Db.query(sql, [])), targets: [{ name: "d1", available: true, semantics, run: (compiled) => runWith(compiled, d1Db.query) }], close: d1Db.close, } From 79d0d722dc820e86f8c8927e062f139bbc2e63e9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:17:35 +0100 Subject: [PATCH 16/24] test(fuzz): a query pinned to one org by its own filter is single-tenant for that org The tenant oracle took "single-tenant" to mean the orgId param's tenant. `OrgId IN ('org_2')` pins a query to org_2, which the builder rightly reports as single-tenant; the oracle called reading org_2 a leak. Where the plan's WHERE filters OrgId itself, any one org is right; otherwise it must still be the param's. Found by a 20x soak on D1; pinned. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/queries.fuzz.test.ts | 10 +++++++--- tests/fuzz/regressions.ts | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/fuzz/queries.fuzz.test.ts b/tests/fuzz/queries.fuzz.test.ts index c836dd8..890cc31 100644 --- a/tests/fuzz/queries.fuzz.test.ts +++ b/tests/fuzz/queries.fuzz.test.ts @@ -33,7 +33,7 @@ const sorted = (rows: Array) => rows.map((r) => JSON.stringify(r)).sort const matchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Plan) => { const known = knownPlanIssue({ target: target.name, plan: input }) if (known !== undefined) skipKnown(known) - return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input)) + return checkRows(target, input.orgId, evaluate(input, target.semantics), compilePlan(dialect, dialect.context, input), compilePlan(dialect, scopeContext(dialect), input), input.where) } const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: UnionPlan) => { @@ -49,11 +49,12 @@ const unionMatchesModel = (dialect: FuzzDialect, target: FuzzTarget, input: Unio evaluateUnion(input, target.semantics), compileUnionPlan(dialect, dialect.context, input), compileUnionPlan(dialect, scopeContext(dialect), input), + input.branches.map((branch) => branch.where), ) } /** Run what compiled and compare its rows with the model's; `scoped` is the same input over the plain tables. */ -const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled) => { +const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResult, result: Compiled, scoped: Compiled, where: unknown) => { if (result._tag !== "Compiled") throw new FuzzFailure(`a valid plan was not compiled (${result._tag})`, { error: result.error }) const { compiled } = result // The scope a real application would get: the same plan over the plain tables. @@ -67,7 +68,10 @@ const checkRows = async (target: FuzzTarget, orgId: string, expected: ModelResul if (expected.ordered) expect(actual).toEqual(want) else expect(sorted(actual)).toEqual(sorted(want)) if (tenantScope === "single-tenant") { - const foreign = [...expected.orgs].filter((org) => org !== orgId) + // The tenant is the `orgId` param, unless the plan also constrains OrgId itself (`OrgId IN ('org_2')` + // pins the query to org_2): then it is single-tenant by that, and any one org is right. + const pinnedElsewhere = /"col":"OrgId"/.test(JSON.stringify(where)) + const foreign = pinnedElsewhere ? [...expected.orgs].slice(1) : [...expected.orgs].filter((org) => org !== orgId) expect(foreign, "a single-tenant query read another org's rows").toEqual([]) } }) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 283f51a..8193925 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -36,6 +36,11 @@ export const pinned: Record> = { { k: "or", style: "method", cs: [{ k: "and", style: "method", cs: [{ k: "const", v: true }, { k: "tenant", side: "orders" }] }, { k: "tenant", side: "orders" }] }, ], }, + // `OrgId IN ('org_2')` pins the query to org_2, not to the orgId param: single-tenant, and right to read org_2. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + where: [{ k: "in", neg: false, e: { k: "col", col: "OrgId" }, values: ["org_2"] }], + }, // pg-fraction-bound-to-integer. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), From 4a7f6d3700a2987c827aa88bfb485cdb7ca58837 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:26:04 +0100 Subject: [PATCH 17/24] fix(clickhouse): cast a union's numeric column around each branch, not inside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inside a branch the cast ran before that branch's DISTINCT, so two UInt64 values past 2^53 (one Float64) became one row. The cast now wraps each branch, SELECT toFloat64(x) AS x, … FROM (), so DISTINCT, GROUP BY and LIMIT see exact values (CodeRabbit). The writes suite now accepts only a QueryBuilderError or a QueryBuilderDefect as the refusal of a clause a dialect lacks, not any defect. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 2 +- docs/unions-and-ctes.md | 5 +++-- src/ch/compile.test.ts | 14 +++++++------- src/ch/compile.ts | 18 +++++++++++------- tests/fuzz/writes.fuzz.test.ts | 4 ++++ 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index e92afff..0296930 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -3,6 +3,6 @@ --- On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and -`Id - 1`, or a `UInt32` and an `Int64` column) is selected as `toFloat64(…)` in each branch. +`Id - 1`, or a `UInt32` and an `Int64` column) is cast with `toFloat64(…)` around each branch. ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index da64a04..54f68cb 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -32,8 +32,9 @@ Unions compile with **`compileUnion`**, not `compile`. > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, > with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose > branches may differ in type (different declared types, or different arithmetic: `Id + 1` is -> `UInt64`, `Id - 1` is `Int64`) is selected as `toFloat64(…)` in every branch. Numbers decode to -> JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical +> `UInt64`, `Id - 1` is `Int64`) is cast with `toFloat64(…)` around every branch +> (`SELECT toFloat64(x) AS x, … FROM ()`), so a branch's own DISTINCT, GROUP BY and LIMIT +> still see exact values. Numbers decode to JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical > expressions are left as they are. Cast other types explicitly when the branches differ. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 90dd204..3cc4f6f 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -164,19 +164,19 @@ describe("CompiledQuery.decodeRows", () => { const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) - expect(compiled.sql).toContain("toFloat64(orders.Id + 1) AS shifted") - expect(compiled.sql).toContain("toFloat64(orders.Id + -1) AS shifted") + // Around each branch, so a branch's own DISTINCT sees the exact values. + expect(compiled.sql.match(/SELECT id, toFloat64\(shifted\) AS shifted, total, name FROM \(/g)).toHaveLength(2) + expect(compiled.sql).toContain("orders.Id + 1 AS shifted") // One declared type in every branch, and non-numbers, stay as they are. - expect(compiled.sql).not.toContain("toFloat64(orders.Id) AS id") - expect(compiled.sql).not.toContain("toFloat64(orders.Amount)") - expect(compiled.sql).not.toContain("toFloat64(orders.Name)") + expect(compiled.sql).not.toContain("toFloat64(id)") + expect(compiled.sql).not.toContain("toFloat64(total)") + expect(compiled.sql).not.toContain("toFloat64(name)") // Different declared types do differ. const mixed = compileUnionUnsafe( CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), {}, ) - expect(mixed.sql).toContain("toFloat64(orders.Id) AS n") - expect(mixed.sql).toContain("toFloat64(orders.Amount) AS n") + expect(mixed.sql.match(/SELECT toFloat64\(n\) AS n FROM \(/g)).toHaveLength(2) }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 8213f78..d69b875 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -718,8 +718,6 @@ function compileInner< params: Params, options?: { selectKeys?: ReadonlyArray - /** Union columns cast with the dialect's `unionNumberCast`; see `unionNumberCasts`. */ - castNumbers?: ReadonlySet skipFormat?: boolean rowSchema?: CompiledQueryRowSchema /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. @@ -924,8 +922,7 @@ function compileInner< } const fragment = expr.toFragment() const [sql, found] = track(() => compileSqlFragment(fragment)) - const numberCast = options?.castNumbers?.has(alias) === true ? currentDialect().unionNumberCast : undefined - return { alias, sql: numberCast !== undefined ? numberCast(castSelectedParam(expr, sql)) : castSelectedParam(expr, sql), found, column: plainColumn(fragment) } + return { alias, sql: castSelectedParam(expr, sql), found, column: plainColumn(fragment) } }) const where = presentWhere.map((c) => raw(rowFilter("WHERE", c))) const having = (state.havingFn ? state.havingFn($) : []) @@ -1452,10 +1449,17 @@ function compileUnionInner, Params extends Re if (state.queries.some((q) => q._state.lock !== undefined)) { return fail(new QueryBuilderDefect({ message: "unionAll: a branch cannot take a row lock; lock in a query over the union instead" }),failedQuery()) } - const castNumbers = unionNumberCasts(state.queries, selectKeys) const subQueries = state.queries.map((q) => - compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes, castNumbers }), + compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) + // Cast around each branch, not inside it: a branch's DISTINCT, GROUP BY and LIMIT still see the + // exact values (two UInt64s past 2^53 are one Float64). + const castNumbers = unionNumberCasts(state.queries, selectKeys) + const numberCast = currentDialect().unionNumberCast + const branchSql = (sql: string): string => + castNumbers.size === 0 || numberCast === undefined + ? sql + : `SELECT ${selectKeys.map((key) => (castNumbers.has(key) ? `${numberCast(quoteIdent(key))} AS ${quoteIdent(key)}` : quoteIdent(key))).join(", ")} FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__branch")}` : ""}` const bounds = new Set( subQueries.flatMap((q) => { const bound = tenantBoundOf(q) @@ -1479,7 +1483,7 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => branchSql(q.sql)).map((q) => (parenthesize ? `(\n${q}\n)` : q)).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = diff --git a/tests/fuzz/writes.fuzz.test.ts b/tests/fuzz/writes.fuzz.test.ts index 49ff3ac..409cad9 100644 --- a/tests/fuzz/writes.fuzz.test.ts +++ b/tests/fuzz/writes.fuzz.test.ts @@ -324,6 +324,10 @@ const writesMatchModel = async (dialect: FuzzDialect, input: WritesInput) => { if (result._tag === "Threw") throw new FuzzFailure("the builder threw instead of failing", context) if (missing !== undefined) { if (result._tag === "Compiled") throw new FuzzFailure(`${missing} compiled for ${dialect.name}, which does not support it`, context) + // A clause the dialect lacks fails with a typed error, or dies with a QueryBuilderDefect (src/ch/errors.ts); nothing else. + if (result._tag !== "Refused" && !(result._tag === "Defect" && result.error.includes("QueryBuilderDefect"))) { + throw new FuzzFailure(`${missing} was not refused with a QueryBuilderError or QueryBuilderDefect`, context) + } continue } if (result._tag !== "Compiled") throw new FuzzFailure(`a supported write was not compiled (${result._tag})`, context) From b0619ae0c8de65c370dc1cc5db32266dba843060 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:30:45 +0100 Subject: [PATCH 18/24] fix(clickhouse): order a union's mixed numeric column by exact keys, never cast its values The previous fix cast such a column to Float64, which an independent review showed corrupts integers past 2^53 wherever the values stay on the server: INSERT ... SELECT from the union stored 9007199254740992 for 9007199254740993, and an IN over it matched nothing. Only the union's ORDER BY was ever broken (a Variant is not a valid sort key), so only the sort key changes now: toFloat64(x), then accurateCastOrNull(x, 'Int128'), which keeps integers past 2^53 in exact order. Checked on ClickHouse 26.8 with UInt64/Int64 values around 2^53 and 2^64 - 1. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 9 ++++---- docs/unions-and-ctes.md | 13 ++++++----- src/ch/compile.test.ts | 20 +++++++--------- src/ch/compile.ts | 32 +++++++++++++------------- src/ch/dialect.ts | 16 +++++++------ 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index 0296930..cb2facb 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -2,7 +2,8 @@ "@maple-dev/effect-orm": patch --- -On ClickHouse, a `unionAll` column of numbers whose branches may differ in type (`Id + 1` and -`Id - 1`, or a `UInt32` and an `Int64` column) is cast with `toFloat64(…)` around each branch. -ClickHouse had made it a `Variant`, and an ORDER BY on it failed with "Data types Variant/Dynamic -are not allowed in ORDER BY keys". Numbers decode to JS numbers either way. +On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1` +and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic +are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by +`toFloat64(x), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the +values themselves are not converted. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 54f68cb..53c08c1 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -30,12 +30,13 @@ Unions compile with **`compileUnion`**, not `compile`. > **Numeric branch types.** ClickHouse has no common type for some numeric pairs (`UInt64` and > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, -> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse a numeric column whose -> branches may differ in type (different declared types, or different arithmetic: `Id + 1` is -> `UInt64`, `Id - 1` is `Int64`) is cast with `toFloat64(…)` around every branch -> (`SELECT toFloat64(x) AS x, … FROM ()`), so a branch's own DISTINCT, GROUP BY and LIMIT -> still see exact values. Numbers decode to JS numbers either way, so this loses nothing the decode keeps. Identical columns and identical -> expressions are left as they are. Cast other types explicitly when the branches differ. +> with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy` +> sorts a numeric column whose branches may differ in type (different declared types, or different +> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `toFloat64(x)` and then +> `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values +> themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them +> exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that +> needs one type, should cast it in each branch. _(Backed by `docs/unions-and-ctes.md > unionAll with an outer ORDER BY`.)_ diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 3cc4f6f..50e1c99 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -159,24 +159,20 @@ describe("CompiledQuery.decodeRows", () => { expect(compiled.rowSchemaMismatch).toEqual({ undeclared: [], unselected: ["missing"] }) }) - it("casts a ClickHouse union's numeric column to one type when its branches may differ", () => { + it("orders a ClickHouse union's mixed numeric column by exact keys, leaving its values alone", () => { // `Id + 1` is UInt64 and `Id - 1` Int64: ClickHouse makes the column a Variant, which ORDER BY refuses. const table = CH.table("orders", { Id: CH.uint32, Amount: CH.int64, Name: CH.string }) const branch = (offset: number) => CH.from(table).select(($) => ({ id: $.Id, shifted: $.Id.add(offset), total: $.Amount, name: $.Name })) - const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "asc"]), {}) - // Around each branch, so a branch's own DISTINCT sees the exact values. - expect(compiled.sql.match(/SELECT id, toFloat64\(shifted\) AS shifted, total, name FROM \(/g)).toHaveLength(2) - expect(compiled.sql).toContain("orders.Id + 1 AS shifted") - // One declared type in every branch, and non-numbers, stay as they are. - expect(compiled.sql).not.toContain("toFloat64(id)") - expect(compiled.sql).not.toContain("toFloat64(total)") - expect(compiled.sql).not.toContain("toFloat64(name)") - // Different declared types do differ. + const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {}) + // The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact. + expect(compiled.sql).not.toContain("toFloat64(orders") + expect(compiled.sql).toContain("ORDER BY toFloat64(shifted) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") + // Different declared types are mixed too; one declared type in every branch is not. const mixed = compileUnionUnsafe( - CH.unionAll(CH.from(table).select(($) => ({ n: $.Id })), CH.from(table).select(($) => ({ n: $.Amount }))), + CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]), {}, ) - expect(mixed.sql.match(/SELECT toFloat64\(n\) AS n FROM \(/g)).toHaveLength(2) + expect(mixed.sql).toContain("ORDER BY toFloat64(n) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/compile.ts b/src/ch/compile.ts index d69b875..afb784b 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -1400,14 +1400,14 @@ const isNumberSchema = (schema: unknown): boolean => { } /** - * The union columns to cast to one numeric type, on a dialect with a - * `unionNumberCast`: every branch selects a number there, and their types may - * differ, because the branches declare different column types, or compute it - * differently (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). + * The union columns a dialect with `unionNumberOrderKeys` orders through those + * keys: every branch selects a number, and their types may differ, because the + * branches declare different column types or compute it differently + * (arithmetic has no declared type: `Id + 1` is UInt64, `Id - 1` Int64). */ -function unionNumberCasts(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { +function mixedNumberColumns(queries: ReadonlyArray>, selectKeys: ReadonlyArray): ReadonlySet { const casts = new Set() - if (currentDialect().unionNumberCast === undefined || queries.length < 2) return casts + if (currentDialect().unionNumberOrderKeys === undefined || queries.length < 2) return casts for (const alias of selectKeys) { const exprs = queries.map((q) => (selectExprsOf(q) as Record | undefined)?.[alias]) const kinds = exprs.map((expr) => { @@ -1452,14 +1452,7 @@ function compileUnionInner, Params extends Re const subQueries = state.queries.map((q) => compileInner(q, params, { skipFormat: true, deferParams, nested: true, selectKeys, enclosingCtes }), ) - // Cast around each branch, not inside it: a branch's DISTINCT, GROUP BY and LIMIT still see the - // exact values (two UInt64s past 2^53 are one Float64). - const castNumbers = unionNumberCasts(state.queries, selectKeys) - const numberCast = currentDialect().unionNumberCast - const branchSql = (sql: string): string => - castNumbers.size === 0 || numberCast === undefined - ? sql - : `SELECT ${selectKeys.map((key) => (castNumbers.has(key) ? `${numberCast(quoteIdent(key))} AS ${quoteIdent(key)}` : quoteIdent(key))).join(", ")} FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__branch")}` : ""}` + const mixedNumbers = mixedNumberColumns(state.queries, selectKeys) const bounds = new Set( subQueries.flatMap((q) => { const bound = tenantBoundOf(q) @@ -1483,7 +1476,7 @@ function compileUnionInner, Params extends Re : "cross-tenant" const parenthesize = currentDialect().clauses.parenthesizeUnionBranches - let sql = subQueries.map((q) => branchSql(q.sql)).map((q) => (parenthesize ? `(\n${q}\n)` : q)).join("\nUNION ALL\n") + let sql = subQueries.map((q) => (parenthesize ? `(\n${q.sql}\n)` : q.sql)).join("\nUNION ALL\n") // Wrap in outer SELECT if ordering/pagination is needed const hasOuter = @@ -1492,7 +1485,14 @@ function compileUnionInner, Params extends Re if (hasOuter) { sql = `SELECT * FROM (\n${sql}\n)${currentDialect().clauses.derivedTableAlias ? ` AS ${quoteIdent("__union")}` : ""}` if (state.outerOrderBySpecs.length > 0) { - sql += `\nORDER BY ${orderByClause(state.outerOrderBySpecs).join(", ")}` + const orderKeys = currentDialect().unionNumberOrderKeys + const clause = orderByClause(state.outerOrderBySpecs).map((rendered, i) => { + const [column, direction] = state.outerOrderBySpecs[i]! + return orderKeys !== undefined && mixedNumbers.has(column) + ? orderKeys(quoteIdent(column)).map((key) => `${key} ${direction.toUpperCase()}`).join(", ") + : rendered + }) + sql += `\nORDER BY ${clause.join(", ")}` } if (state.outerLimitValue != null) { sql += `\nLIMIT ${compileSqlFragment(rowCount("limit", state.outerLimitValue)!)}` diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index d77dee7..9a8d7b9 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -152,12 +152,14 @@ export interface Dialect extends Omit { */ readonly functions?: string /** - * Casts a numeric UNION ALL column whose branches may have different types - * to one type. ClickHouse has no common type for `UInt64` and `Int64` (or - * `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY - * refuses. Absent: the dialect unifies numeric branches itself. + * The ORDER BY keys for a numeric UNION ALL column whose branches may have + * different types. ClickHouse has no common type for `UInt64` and `Int64` + * (or `Int64` and `Float64`): the column is a `Variant`, which ORDER BY + * refuses as a key. The values are left as they are (an INSERT ... SELECT or + * an IN over the union keeps them exact); only the sort key is converted. + * Absent: the dialect orders such a column as it is. */ - readonly unionNumberCast?: (sql: string) => string + readonly unionNumberOrderKeys?: (column: string) => ReadonlyArray } /** ClickHouse, with params written into the SQL as literals. The default. */ @@ -180,8 +182,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", - // Every numeric ClickHouse type decodes to a JS number (CHNumber), so Float64 loses nothing the decode keeps. - unionNumberCast: (sql) => `toFloat64(${sql})`, + // Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. + unionNumberOrderKeys: (column) => [`toFloat64(${column})`, `accurateCastOrNull(${column}, 'Int128')`], } // The dialect of the enclosing compile, beside the syntax installed for the From 77eec2ad541a06d6edf344c5ac1ed2d71d1afb12 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:32:25 +0100 Subject: [PATCH 19/24] fix(sqlite): refuse timestamps outside 0000-9999; fraction rule as a dialect flag From an independent review: - A timestamp past year 9999 (new Date(8.64e15), a common max-date sentinel) was written as '+275760-09 13T00:00:00.', which then failed to decode, and `at > year 10000` bound text that sorts before every real timestamp, so it matched every row. Inserts, comparisons and params now refuse it at compile time; sqliteTimestampText throws. - `integer > 9.75` keyed off the dialect name, so a profile spread from sqliteDialect under another name lost it. It is the dialect flag integerComparesFraction now. Co-Authored-By: Claude Opus 5.5 --- src/ch/dialect.ts | 1 + src/ch/expr.ts | 2 +- src/sql/sql-syntax.ts | 6 ++++++ src/sqlite/dialect.ts | 1 + src/sqlite/sqlite.test.ts | 19 +++++++++++++++++++ src/sqlite/timestamp.ts | 18 +++++++++++++++++- 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b01a9..bf38def 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -236,6 +236,7 @@ const checkedSyntax = (dialect: Dialect): SqlSyntax => ({ ilike: dialect.ilike, likeEscape: dialect.likeEscape, inexactNumberText: dialect.inexactNumberText, + integerComparesFraction: dialect.integerComparesFraction, quoteIdent: dialect.quoteIdent, quoteString: (value) => checked(dialect, dialect.quoteString(value), "a string literal"), literal: (value, context) => checkedLiteral(dialect, value, context), diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 774b9d5..6c2f84e 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -370,7 +370,7 @@ const sqliteFraction = (columnType: CHType, value: unknown): b typeof value === "number" && Number.isFinite(value) && !Number.isInteger(value) && - (activeSqlSyntax()?.dialect === "sqlite" || activeSqlSyntax()?.dialect === "d1") + activeSqlSyntax()?.integerComparesFraction === true const POSTGRES_INTEGERS = new Set(["int2", "int4", "int8", "smallint", "integer", "bigint"]) diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index 1bdd78a..e87bb68 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -24,6 +24,12 @@ export interface SqlSyntax { * double (SQLite), so a fraction must be bound rather than written as text. */ readonly inexactNumberText?: boolean + /** + * True when an integer column compares with a fraction numerically (SQLite's + * type affinity), so `integer > 9.75` writes the fraction instead of refusing + * it through the column's whole-number codec. + */ + readonly integerComparesFraction?: boolean /** One identifier (a column, alias, table or schema name), quoted as needed. */ readonly quoteIdent: (name: string) => string /** A string as a quoted literal. */ diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index ccaef3f..359a847 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -37,6 +37,7 @@ export const sqliteDialect: Dialect = { ilike: false, likeEscape: true, inexactNumberText: true, + integerComparesFraction: true, quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, quoteString, literal, diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index 1ba4e77..fe5ff92 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -117,6 +117,25 @@ it("compares timestamps with the text SQLite's own functions write, and reads zo expect(written.parameters).toContain("2026-10-01 08:00:00.500") }) +it("refuses a timestamp outside 0000-9999 instead of writing text that is no timestamp", () => { + const Events = SQLite.table("events", { id: SQLite.integer, at: SQLite.timestamp }) + const refused = (effect: Effect.Effect) => { + const exit = Effect.runSyncExit(effect) + return Exit.isFailure(exit) && /outside 0000-9999/.test(String(Cause.squash(exit.cause))) + } + const far = DateTime.makeUnsafe(new Date(8.64e15)) + expect(refused(SQLite.compile(SQLite.insertInto(Events).values({ id: 1, at: far })))).toBe(true) + expect(refused(SQLite.compile(SQLite.from(Events).select("id").where(($) => [$.at.gt(DateTime.makeUnsafe("+010000-01-01T00:00:00Z"))])))).toBe(true) + expect(refused(SQLite.compile(SQLite.from(Events).select("id").where(($) => [$.at.gt(SQLite.param.dateTime("t"))]), { t: far }))).toBe(true) + expect(() => SQLite.sqliteTimestampText(8.64e15)).toThrow(/outside 0000-9999/) +}) + +it("compares a fraction with an integer column on a profile spread from sqliteDialect", () => { + const profile = { ...SQLite.sqliteDialect, name: "my-sqlite" } + const compiled = SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.id.gt(9.75)]), {}, { dialect: profile }) + expect(compiled.parameters).toEqual([9.75]) +}) + it("refuses an integer past 2^53 instead of decoding a different number", async () => { const exit = await withDatabase((db) => Effect.gen(function* () { // node:sqlite refuses an unsafe integer itself; D1 hands one over (see d1.test.ts), where the codec refuses it. diff --git a/src/sqlite/timestamp.ts b/src/sqlite/timestamp.ts index e30b08f..8456055 100644 --- a/src/sqlite/timestamp.ts +++ b/src/sqlite/timestamp.ts @@ -10,6 +10,8 @@ import { pgTimestampToIso, timestampLiteral } from "../pg/types" */ export const sqliteTimestampText = (epochMillis: number): string => { const iso = new Date(epochMillis).toISOString() + // Past year 9999 toISOString writes `+010000-…`, which is no timestamp and sorts before every real one. + if (!/^\d{4}-/.test(iso)) throw new RangeError(`${iso} is outside 0000-9999, which SQLite's text timestamps cover`) return `${iso.slice(0, 10)} ${iso.slice(11, 23)}` } @@ -39,6 +41,20 @@ export const SqliteTimestamp: Schema.Codec = Schema.String }), ) +const epochOf = (value: unknown): number | undefined => + DateTime.isDateTime(value) ? DateTime.toEpochMillis(value) : value instanceof Date ? value.getTime() : typeof value === "string" ? Date.parse(pgTimestampToIso(value)) : undefined + +/** Refused before encoding: a value outside 0000-9999 has no SQLite timestamp text. */ +const literalInRange = Schema.makeFilter( + (value: unknown) => { + const millis = epochOf(value) + if (millis === undefined || Number.isNaN(millis)) return undefined + const year = new Date(millis).getUTCFullYear() + return year >= 0 && year <= 9999 ? undefined : `year ${year} is outside 0000-9999, which SQLite's text timestamps cover` + }, + { title: "sqliteTimestampRange" }, +) + /** What a timestamp comparison or param accepts (a `DateTime.Utc`, a `Date`, timestamp text), written in SQLite's form. */ -export const SqliteTimestampLiteral = timestampLiteral(sqliteTimestampText) +export const SqliteTimestampLiteral = timestampLiteral(sqliteTimestampText).pipe(Schema.check(literalInRange)) From 04c6de3366415b1372f37be5695a1e297f381e48 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:42:38 +0100 Subject: [PATCH 20/24] fix(sqlite): refuse an out-of-range timestamp with a typed error, never a throw The range check ran after a union member's encoder, which threw a RangeError for a Date before year 0, so compile died instead of failing with a QueryBuilderError (found by the hostile-values fuzz property). Each member of the timestamp codec now checks the range on its decoded side, and the encoder never throws; dateTimeLiteral fails with a typed error too. Pinned. Co-Authored-By: Claude Opus 5.5 --- src/sqlite/dialect.ts | 13 ++++-- src/sqlite/timestamp.ts | 92 ++++++++++++++++++++++++++------------- tests/fuzz/regressions.ts | 2 + 3 files changed, 73 insertions(+), 34 deletions(-) diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index 359a847..f4a79f9 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -3,8 +3,7 @@ import { noTransactions, type Dialect } from "../ch/dialect" import { QueryBuilderError } from "../ch/errors" import { fail } from "../ch/failure" import { PARAM_MARKER_PREFIX } from "../ch/param" -import { timestampLiteral } from "../pg/types" -import { SqliteTimestampLiteral, sqliteTimestampText } from "./timestamp" +import { SqliteTimestampLiteral, sqliteTimestampLiteral, sqliteTimestampText } from "./timestamp" const hex = (value: Uint8Array): string => Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("") @@ -41,7 +40,13 @@ export const sqliteDialect: Dialect = { quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, quoteString, literal, - dateTimeLiteral: (value) => quoteString(sqliteTimestampText(DateTime.toEpochMillis(value))), + dateTimeLiteral: (value) => { + try { + return quoteString(sqliteTimestampText(DateTime.toEpochMillis(value))) + } catch (error) { + return fail(new QueryBuilderError({ code: "InvalidLiteral", message: (error as Error).message }), "NULL") + } + }, params: { _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 32766, nulInText: false }, clauses: { format: false, @@ -59,7 +64,7 @@ export const sqliteDialect: Dialect = { }, paramCodecs: { dateTime: SqliteTimestampLiteral, - dateTimeSeconds: timestampLiteral((millis) => sqliteTimestampText(Math.floor(millis / 1000) * 1000)), + dateTimeSeconds: sqliteTimestampLiteral((millis) => Math.floor(millis / 1000) * 1000), }, transactions: { support: "full", diff --git a/src/sqlite/timestamp.ts b/src/sqlite/timestamp.ts index 8456055..ae1a048 100644 --- a/src/sqlite/timestamp.ts +++ b/src/sqlite/timestamp.ts @@ -1,6 +1,6 @@ // SQLite timestamp text: the form `SQLite.timestamp` writes and reads. import { DateTime, Schema, SchemaGetter } from "effect" -import { pgTimestampToIso, timestampLiteral } from "../pg/types" +import { pgTimestampToIso } from "../pg/types" /** * An instant as SQLite's own text form, `YYYY-MM-DD HH:MM:SS.SSS` in UTC: what @@ -9,52 +9,84 @@ import { pgTimestampToIso, timestampLiteral } from "../pg/types" * one form; this is the form SQLite's own functions produce (see `timestampNow`). */ export const sqliteTimestampText = (epochMillis: number): string => { + if (!inRange(epochMillis)) throw new RangeError(`${new Date(epochMillis).toISOString()} is outside 0000-9999, which SQLite's text timestamps cover`) + return textOf(epochMillis) +} + +/** The text, for an instant already checked to be in range. */ +const textOf = (epochMillis: number): string => { const iso = new Date(epochMillis).toISOString() - // Past year 9999 toISOString writes `+010000-…`, which is no timestamp and sorts before every real one. - if (!/^\d{4}-/.test(iso)) throw new RangeError(`${iso} is outside 0000-9999, which SQLite's text timestamps cover`) return `${iso.slice(0, 10)} ${iso.slice(11, 23)}` } -/** Years SQLite's date functions and the text form cover: four digits, no sign. */ -const inSqliteRange = Schema.makeFilter( - (value: DateTime.Utc) => { - const year = new Date(DateTime.toEpochMillis(value)).getUTCFullYear() - return year >= 0 && year <= 9999 ? undefined : `year ${year} is outside 0000-9999, which SQLite's text timestamps cover` - }, +/** Years SQLite's date functions and the text form cover: four digits, no sign. Past 9999, toISOString writes `+010000-…`. */ +const inRange = (epochMillis: number): boolean => { + const year = new Date(epochMillis).getUTCFullYear() + return year >= 0 && year <= 9999 +} +const rangeMessage = (epochMillis: number) => `year ${new Date(epochMillis).getUTCFullYear()} is outside 0000-9999, which SQLite's text timestamps cover` + +// Each check sits on the decoded side, so a value outside the range is refused before an encoder sees it. +const dateTimeInRange = Schema.makeFilter((value: DateTime.Utc) => (inRange(DateTime.toEpochMillis(value)) ? undefined : rangeMessage(DateTime.toEpochMillis(value))), { + title: "sqliteTimestampRange", +}) +const dateInRange = Schema.makeFilter( + (value: Date) => (Number.isNaN(value.getTime()) ? "an invalid Date" : inRange(value.getTime()) ? undefined : rangeMessage(value.getTime())), { title: "sqliteTimestampRange" }, ) const isTimestampText = Schema.makeFilter( - (value: string) => - Number.isNaN(Date.parse(pgTimestampToIso(value))) - ? `\`${value}\` is not a timestamp: expected \`YYYY-MM-DD HH:MM:SS[.SSS]\` or ISO-8601` - : undefined, + (value: string) => { + const millis = Date.parse(pgTimestampToIso(value)) + if (Number.isNaN(millis)) return `\`${value}\` is not a timestamp: expected \`YYYY-MM-DD HH:MM:SS[.SSS]\` or ISO-8601` + return inRange(millis) ? undefined : rangeMessage(millis) + }, { title: "sqliteTimestamp" }, ) /** Reads SQLite's form and ISO-8601 (zoneless text is UTC); writes SQLite's form. */ export const SqliteTimestamp: Schema.Codec = Schema.String.pipe( Schema.check(isTimestampText), - Schema.decodeTo(Schema.DateTimeUtc.pipe(Schema.check(inSqliteRange)), { + Schema.decodeTo(Schema.DateTimeUtc.pipe(Schema.check(dateTimeInRange)), { decode: SchemaGetter.transform((value: string) => DateTime.makeUnsafe(pgTimestampToIso(value))), - encode: SchemaGetter.transform((value: DateTime.Utc) => sqliteTimestampText(DateTime.toEpochMillis(value))), + encode: SchemaGetter.transform((value: DateTime.Utc) => textOf(DateTime.toEpochMillis(value))), }), ) -const epochOf = (value: unknown): number | undefined => - DateTime.isDateTime(value) ? DateTime.toEpochMillis(value) : value instanceof Date ? value.getTime() : typeof value === "string" ? Date.parse(pgTimestampToIso(value)) : undefined - -/** Refused before encoding: a value outside 0000-9999 has no SQLite timestamp text. */ -const literalInRange = Schema.makeFilter( - (value: unknown) => { - const millis = epochOf(value) - if (millis === undefined || Number.isNaN(millis)) return undefined - const year = new Date(millis).getUTCFullYear() - return year >= 0 && year <= 9999 ? undefined : `year ${year} is outside 0000-9999, which SQLite's text timestamps cover` - }, - { title: "sqliteTimestampRange" }, -) +/** + * What a timestamp comparison or param accepts (a `DateTime.Utc`, a `Date`, + * timestamp text), written in SQLite's form after `adjust` (truncation to + * seconds, for `param.dateTimeSeconds`). + */ +export const sqliteTimestampLiteral = (adjust: (epochMillis: number) => number = (millis) => millis): Schema.Codec => + Schema.Union([ + Schema.String.pipe( + Schema.check(isTimestampText), + Schema.decodeTo(Schema.DateTimeUtc.pipe(Schema.check(dateTimeInRange)), { + decode: SchemaGetter.transform((value: string) => DateTime.makeUnsafe(pgTimestampToIso(value))), + encode: SchemaGetter.transform((value: DateTime.Utc) => textOf(adjust(DateTime.toEpochMillis(value)))), + }), + ), + Schema.String.pipe( + Schema.decodeTo(Schema.Date.pipe(Schema.check(dateInRange)), { + decode: SchemaGetter.transform((value: string) => new Date(pgTimestampToIso(value))), + encode: SchemaGetter.transform((value: Date) => textOf(adjust(value.getTime()))), + }), + ), + Schema.String.pipe( + Schema.check(isTimestampText), + Schema.decodeTo(Schema.String.pipe(Schema.check(isTimestampText)), { + decode: SchemaGetter.transform((value: string) => value), + encode: SchemaGetter.transform((value: string) => textOf(adjust(Date.parse(pgTimestampToIso(value))))), + }), + ), + ]) as Schema.Codec -/** What a timestamp comparison or param accepts (a `DateTime.Utc`, a `Date`, timestamp text), written in SQLite's form. */ -export const SqliteTimestampLiteral = timestampLiteral(sqliteTimestampText).pipe(Schema.check(literalInRange)) +export const SqliteTimestampLiteral = sqliteTimestampLiteral() +/** + * A `defaultExpr` for a `timestamp` column: the current time in the form the + * column writes. `CURRENT_TIMESTAMP` writes no milliseconds, so its values + * compare as earlier than the same instant written by the builder. + */ +export const timestampNow = "(strftime('%Y-%m-%d %H:%M:%f', 'now'))" diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index c65e60f..be171a3 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -90,6 +90,8 @@ export const pinned: Record> = { { dialect: "clickhouse", column: "OrgId", site: "add", value: undefined }, { dialect: "clickhouse", column: "OrgId", site: "add", value: new Date(Number.NaN) }, ], + // A Date before year 0 reached a timestamp encoder that threw instead of failing. + "hostile values are refused, never thrown (sqlite)": [{ dialect: "sqlite", column: "Created", site: "gt", value: new Date("-000001-12-31T23:59:59.999Z") }], "hostile values are refused, never thrown (postgres)": [ { dialect: "postgres", column: "OrgId", site: "param.dateTime", value: "" }, { dialect: "postgres", column: "Created", site: "param.of", value: "2026-13-45 99:99:99" }, From 6ca87b00d337589bb027f822a1d6dcc639efd919 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:46:47 +0100 Subject: [PATCH 21/24] fix(clickhouse): order union numbers with CAST, which ClickHouse 26.2 accepts on a Variant toFloat64 refuses a Variant on ClickHouse 26.2 (CI); CAST(x AS Nullable(Float64)) works on 26.2 and 26.8, Variant or not. Checked both versions with the full suite and every fuzz property at 20x. Co-Authored-By: Claude Opus 5.5 --- .changeset/clickhouse-union-numbers.md | 2 +- docs/unions-and-ctes.md | 2 +- src/ch/compile.test.ts | 4 ++-- src/ch/dialect.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.changeset/clickhouse-union-numbers.md b/.changeset/clickhouse-union-numbers.md index cb2facb..dc4ec1e 100644 --- a/.changeset/clickhouse-union-numbers.md +++ b/.changeset/clickhouse-union-numbers.md @@ -5,5 +5,5 @@ On ClickHouse, a `unionAll` ordered by a numeric column whose branches may differ in type (`Id + 1` and `Id - 1`, or a `UInt32` and an `Int64` column) no longer fails with "Data types Variant/Dynamic are not allowed in ORDER BY keys". The union's ORDER BY sorts such a column by -`toFloat64(x), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the +`CAST(x AS Nullable(Float64)), accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order; the values themselves are not converted. diff --git a/docs/unions-and-ctes.md b/docs/unions-and-ctes.md index 53c08c1..4d591fc 100644 --- a/docs/unions-and-ctes.md +++ b/docs/unions-and-ctes.md @@ -32,7 +32,7 @@ Unions compile with **`compileUnion`**, not `compile`. > `Int64`, `Int64` and `Float64`): it makes the column a `Variant`, which ORDER BY refuses, or, > with `use_variant_as_common_type = 0`, rejects the union. So on ClickHouse the union's `orderBy` > sorts a numeric column whose branches may differ in type (different declared types, or different -> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `toFloat64(x)` and then +> arithmetic: `Id + 1` is `UInt64`, `Id - 1` is `Int64`) by `CAST(x AS Nullable(Float64))` and then > `accurateCastOrNull(x, 'Int128')`, which keeps integers past 2^53 in exact order. The values > themselves are not converted, so an `INSERT ... SELECT` or an `IN` over the union keeps them > exact. A query over the union (`fromUnion`) that orders by such a column, or any other use that diff --git a/src/ch/compile.test.ts b/src/ch/compile.test.ts index 50e1c99..d67fe58 100644 --- a/src/ch/compile.test.ts +++ b/src/ch/compile.test.ts @@ -166,13 +166,13 @@ describe("CompiledQuery.decodeRows", () => { const compiled = compileUnionUnsafe(CH.unionAll(branch(1), branch(-1)).orderBy(["shifted", "desc"], ["id", "asc"], ["name", "asc"]), {}) // The values are not cast: an INSERT ... SELECT or IN over the union keeps integers past 2^53 exact. expect(compiled.sql).not.toContain("toFloat64(orders") - expect(compiled.sql).toContain("ORDER BY toFloat64(shifted) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") + expect(compiled.sql).toContain("ORDER BY CAST(shifted AS Nullable(Float64)) DESC, accurateCastOrNull(shifted, 'Int128') DESC, id ASC, name ASC") // Different declared types are mixed too; one declared type in every branch is not. const mixed = compileUnionUnsafe( CH.unionAll(CH.from(table).select(($) => ({ n: $.Id, m: $.Amount })), CH.from(table).select(($) => ({ n: $.Amount, m: $.Amount }))).orderBy(["n", "asc"], ["m", "asc"]), {}, ) - expect(mixed.sql).toContain("ORDER BY toFloat64(n) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") + expect(mixed.sql).toContain("ORDER BY CAST(n AS Nullable(Float64)) ASC, accurateCastOrNull(n, 'Int128') ASC, m ASC") }) it.effect("decodes rows with the declared schema for handwritten SQL", () => diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 9a8d7b9..07ec134 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -182,8 +182,8 @@ export const clickhouseDialect: Dialect = { }, transactions: noTransactions, functions: "clickhouse", - // Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. - unionNumberOrderKeys: (column) => [`toFloat64(${column})`, `accurateCastOrNull(${column}, 'Int128')`], + // CAST, not toFloat64: ClickHouse 26.2's toFloat64 refuses a Variant. Float64 orders every number; the Int128 tiebreak keeps integers past 2^53 (one Float64) in exact order. + unionNumberOrderKeys: (column) => [`CAST(${column} AS Nullable(Float64))`, `accurateCastOrNull(${column}, 'Int128')`], } // The dialect of the enclosing compile, beside the syntax installed for the From cfeaf59c55e143f93b434dd7f0dab99b6c40519c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:55:46 +0100 Subject: [PATCH 22/24] test(fuzz): rerun ClickHouse's one-argument and() bug without the rewrite, rows still checked The known-issue matcher for this ClickHouse server bug covered two shapes; a 20x soak on 26.2 found a third (`(a OR 0 OR a) AND (a OR 0)`). Rather than chase shapes, a ClickHouse target that gets exactly this error runs the query again with optimize_extract_common_expressions = 0, and its rows are compared with the model as usual: these inputs are now checked, not skipped. The matcher and its helpers are gone. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/dialects/clickhouse.ts | 11 +++++++- tests/fuzz/known-issues.ts | 47 ++----------------------------- tests/fuzz/regressions.ts | 2 +- 3 files changed, 13 insertions(+), 47 deletions(-) diff --git a/tests/fuzz/dialects/clickhouse.ts b/tests/fuzz/dialects/clickhouse.ts index f130294..a93962b 100644 --- a/tests/fuzz/dialects/clickhouse.ts +++ b/tests/fuzz/dialects/clickhouse.ts @@ -37,8 +37,17 @@ export const clickhouseText = async ( } } +/** + * A ClickHouse server bug, not the builder's: the common-expression rewrite can + * leave a one-argument and() (`a AND ((b AND a) OR a)`, `(a OR 0 OR a) AND (a OR 0)`, + * and more shapes than a matcher keeps up with), which fails with this error. + * The query is run again with the rewrite off, and its rows still checked. + */ +const SERVER_AND_BUG = /Number of arguments for function "and" should be at least 2/ + const runClickHouse = async (compiled: CH.CompiledQuery, settings: Record): Promise => { - const { ok, text } = await clickhouseText(compiled.sql, settings) + let { ok, text } = await clickhouseText(compiled.sql, settings) + if (!ok && SERVER_AND_BUG.test(text)) ({ ok, text } = await clickhouseText(compiled.sql, { ...settings, optimize_extract_common_expressions: "0" })) if (!ok) return { _tag: "DatabaseError", message: text.trim() } const wire = text .split("\n") diff --git a/tests/fuzz/known-issues.ts b/tests/fuzz/known-issues.ts index e487af7..e47e536 100644 --- a/tests/fuzz/known-issues.ts +++ b/tests/fuzz/known-issues.ts @@ -41,7 +41,6 @@ export const knownHostileIssue = (input: HostileInput): string | undefined => // Query plans (queries.fuzz.test.ts) -import { chKind } from "./model" import type { Plan, UnionPlan } from "./plan" export interface PlanInput { @@ -50,44 +49,7 @@ export interface PlanInput { readonly plan: Plan } -/** Every object in a plan, depth-first: plans are plain data, so this reaches every expression. */ -const nodes = function* (value: unknown): Generator> { - if (Array.isArray(value)) for (const item of value) yield* nodes(item) - else if (value !== null && typeof value === "object") { - yield value as Record - for (const item of Object.values(value)) yield* nodes(item) - } -} - -/** The conjuncts a WHERE list ANDs together, flattening nested ANDs. */ -const conjuncts = (conds: ReadonlyArray): Array => - conds.flatMap((c: any) => (c?.k === "and" ? conjuncts(c.cs) : [c])) -const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) - -const planIssues: ReadonlyArray> = [ - { - id: "clickhouse-server-and-absorption", - reason: - "A ClickHouse 25.8 server bug, not the builder: `WHERE a AND ((b AND a) OR a)` or `a AND ((a OR a) OR a)` fails with " + - "`Number of arguments for function \"and\" should be at least 2` — the common-expression rewrite leaves a " + - "one-argument and(). `SETTINGS optimize_extract_common_expressions = 0` avoids it.", - matches: ({ target, plan }) => { - if (!target.startsWith("clickhouse")) return false - const top = conjuncts(plan.where) - const disjuncts = (c: any): Array => (c?.k === "or" ? c.cs.flatMap(disjuncts) : [c]) - // Fails when every disjunct of a nested OR contains a top-level conjunct `w` (as `w`, or inside an AND): - // `w AND ((x AND w) OR w)`, `w AND ((w OR w) OR w)`. `w AND (w OR w)` and `w AND ((x AND w) OR y)` run. - return [...nodes(plan)].some( - (node) => - node.k === "or" && - (node.cs as Array).some((d) => d?.k === "or" || d?.k === "and") && - top.some((w) => disjuncts(node).every((d) => same(d, w) || (d?.k === "and" && conjuncts(d.cs).some((x) => same(x, w))))), - ) - }, - }, - - -] +const planIssues: ReadonlyArray> = [] export const knownPlanIssue = (input: PlanInput): string | undefined => firstMatch(planIssues, input) @@ -98,11 +60,6 @@ export interface UnionInput { readonly plan: UnionPlan } -/** The expression a branch selects under `alias`. */ -const selected = (branch: Plan, alias: string) => (branch.shape.k === "rows" ? branch.shape.select.find(([a]) => a === alias)?.[1] : undefined) - -const unionIssues: ReadonlyArray> = [ - -] +const unionIssues: ReadonlyArray> = [] export const knownUnionIssue = (input: UnionInput): string | undefined => firstMatch(unionIssues, input) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 8193925..05e77f2 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -28,7 +28,7 @@ export const pinned: Record> = { }), // fromquery-select-all. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, - // clickhouse-server-and-absorption. + // A ClickHouse server bug (a one-argument and()): run again without the rewrite, rows still checked (dialects/clickhouse.ts). { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), where: [ From 2aa184fe97920d51914a883c491134444e017a4a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:56:35 +0100 Subject: [PATCH 23/24] test: retry a D1 request that never reached workerd Under a 20x soak Miniflare's local socket occasionally drops ('fetch failed'). Like the ClickHouse helper, the D1 helper retries transport errors only; a database error still fails. Co-Authored-By: Claude Opus 5.5 --- tests/sqlite-support.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/sqlite-support.ts b/tests/sqlite-support.ts index 599065a..3e74673 100644 --- a/tests/sqlite-support.ts +++ b/tests/sqlite-support.ts @@ -3,6 +3,23 @@ import { DatabaseSync } from "node:sqlite" import { Miniflare } from "miniflare" +/** + * Retry what failed to reach workerd ("fetch failed": a dropped local socket + * under a soak's load), as the ClickHouse helper retries a dropped request. A + * database error is not retried: it reached the database and is the finding. + */ +export const retryTransport = async (run: () => Promise): Promise => { + for (let attempt = 1; ; attempt++) { + try { + return await run() + } catch (error) { + const message = error instanceof Error ? `${error.message} ${String((error as { cause?: unknown }).cause ?? "")}` : String(error) + if (attempt >= 5 || !/fetch failed|ECONNRESET|ECONNREFUSED|socket hang up/.test(message)) throw error + await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** attempt)) + } + } +} + export type RawQuery = (sql: string, parameters: ReadonlyArray) => Promise>> /** An in-memory SQLite (Node's built-in driver, the one `@effect/sql-sqlite-node` uses). */ @@ -31,7 +48,7 @@ export const d1Database = (): { readonly query: RawQuery; readonly close: () => return { query: async (sql, parameters) => { const { db } = await start() - return (await db.prepare(sql).bind(...parameters).all()).results + return retryTransport(async () => (await db.prepare(sql).bind(...parameters).all()).results) }, close: async () => { if (started !== undefined) await (await started).mf.dispose() From 433213658d8c3a8014ab523d74e94c581399086b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 25 Oct 2026 19:57:06 +0100 Subject: [PATCH 24/24] test(fuzz): pin the and() server-bug shape ClickHouse 26.2 hit Fails with TOO_FEW_ARGUMENTS_FOR_FUNCTION if the rerun without the rewrite is removed; passes, rows checked, with it. Co-Authored-By: Claude Opus 5.5 --- tests/fuzz/regressions.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/fuzz/regressions.ts b/tests/fuzz/regressions.ts index 05e77f2..5f0b92d 100644 --- a/tests/fuzz/regressions.ts +++ b/tests/fuzz/regressions.ts @@ -28,6 +28,21 @@ export const pinned: Record> = { }), // fromquery-select-all. { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), wrap: "all" }, + // The same server bug in a shape the old matcher missed (ClickHouse 26.2): `(a OR 0 OR a) AND (a OR 0)`. + { + ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }), + orgId: "org_3", + where: [ + { + k: "and", + style: "method", + cs: [ + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "const", v: false }, { k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }] }, + { k: "or", style: "method", cs: [{ k: "null", op: "isNotNull", e: { k: "col", col: "Note" } }, { k: "in", neg: false, e: { k: "str", v: "" }, values: [] }] }, + ], + }, + ], + }, // A ClickHouse server bug (a one-argument and()): run again without the rewrite, rows still checked (dialects/clickhouse.ts). { ...rows({ k: "rows", select: [["id", { k: "col", col: "Id" }]], distinct: false, orderBy: [] }),