diff --git a/.changeset/database-test-layer.md b/.changeset/database-test-layer.md new file mode 100644 index 0000000..d1b6653 --- /dev/null +++ b/.changeset/database-test-layer.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`Db.layerTest({ dialect })` from `/database` is a `Database` with no server, for unit tests. It also gives `TestDatabase`: `statements` holds each statement it was sent (SQL and parameters, transaction control included), and `reply(...)` queues the rows, or the driver `SqlError`, the next statements get. With no reply queued, a statement gets `respond(statement)` from the options, or no rows. The layer is `fromSqlClient` over a fake `SqlClient`, so compiling, row decoding by the query's own schema, `requireTenantScope` and transactions run as they do against a server. diff --git a/docs/README.md b/docs/README.md index 7a61d11..95690e2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ Roughly in reading order. | [Postgres](./postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./migrations.md) | DDL from `CH.table` / `PG.table`, `materializedView`, `effect-orm generate`, applying migrations, adopting drizzle-kit | | [Statements and transactions](./database.md) | `Database` over your `SqlClient`: `run`, `execute`, `transaction`, retry | +| [Testing your code](./testing.md#testing-your-own-code-layertest) | `layerTest`: a `Database` with no server for unit tests | ## Reference diff --git a/docs/database.md b/docs/database.md index 2e0cccd..b8fa7f5 100644 --- a/docs/database.md +++ b/docs/database.md @@ -92,6 +92,7 @@ Calling `withdraw(1, 30)` outside `transfer` does not compile: `requireTransacti | `retryContention(options?)` | Re-run on serialization failure or deadlock | | `Transaction` | The open transaction (`depth`, `isolationLevel`, `accessMode`) | | `isContention(error)` | Whether an error is a serialization failure or deadlock | +| `layerTest(options)`, `TestDatabase` | A `Database` with no server, for unit tests ([Testing](./testing.md#testing-your-own-code-layertest)) | `run`, `query`, `execute`, `transaction` and `retryContention` are also methods on `DatabaseApi`. diff --git a/docs/reference.md b/docs/reference.md index abc2d46..727d731 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -514,6 +514,7 @@ placeholders, double-quoted identifiers. | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | +| `layerTest`, `TestDatabase`, `TestDatabaseApi`, `TestDatabaseOptions`, `TestReply`, `TestStatement` — a `Database` with no server, for unit tests | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | | `defineSuite`, `query`, `caseFromCompiled`, `runSuite`, `compareRuns`, `compareBudgets` | `/benchmark` | diff --git a/docs/testing.md b/docs/testing.md index 708b12e..9530093 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -144,3 +144,72 @@ and [testing guide](https://clickhouse.com/docs/resources/develop-contribute/con are useful sources of additional edge cases. Port relevant operations into the DSL and retain provenance when copying fixtures. Executing upstream SQL unchanged would not validate the builder. The current small fixtures were written for this package. + +## Testing your own code: `layerTest` + +`Db.layerTest(options)` from `/database` is a `Database` with no server, for the unit tests of +code that runs queries. Provide it in place of your real `Database` layer; no module mocks are +needed. It also gives `TestDatabase`, which records what was sent and holds the replies. + +```ts title="database-test-layer.ts" +import { Effect } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Accounts = PG.table("accounts", { columns: { id: PG.int4, org: PG.text, balance: PG.int8 }, primaryKey: ["id"] }) + +// The code under test: it knows only `Database`. +const balances = (org: string) => + Db.run( + PG.from(Accounts) + .select("id", "balance") + .where(($) => [$.org.eq(org)]), + ) + +export const result = await Effect.runPromise( + Effect.gen(function* () { + const test = yield* Db.TestDatabase + // Rows as the server sends them: Postgres returns int8 as a string. + yield* test.reply([{ id: 1, balance: "70" }]) + const rows = yield* balances("org_1") + const [sent] = yield* test.statements + return { rows, parameters: sent?.parameters } + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), +) +// { rows: [{ id: 1, balance: 70 }], parameters: ["org_1"] } +``` + +Only the server is replaced. The layer is `fromSqlClient` over a fake `SqlClient`, so the rest +runs as in production: + +- `run` compiles for the dialect you give. `statements` holds the SQL and `parameters` exactly + as a server would get them. +- Rows are decoded by the query's own row schema. A row that does not match fails with + `CompiledQueryDecodeError`, as it would from a server. +- `transaction` records `BEGIN`, `COMMIT`, `ROLLBACK` and savepoints in `statements`. They + take no reply. On `clickhouseDialect` it fails with `TransactionUnsupported`. +- `requireTenantScope: true` refuses an unscoped query, as on `fromSqlClient`. + +Each statement takes the next queued reply. With no reply queued, it gets `respond(statement)` +from the options, or no rows. A reply is an array of rows, or a driver `SqlError` from +`effect/sql/SqlError`. A driver error becomes the same `DatabaseError` a real driver's would, +with its `reason` and `sqlState`: + +```ts +yield* test.reply( + new SqlError({ reason: new UniqueViolation({ cause: { code: "23505" }, constraint: "accounts_pkey" }) }), +) +const error = yield* Effect.flip(insertAccount) // DatabaseError, reason "UniqueViolation", sqlState "23505" +``` + +| Export | What it is | +| --- | --- | +| `layerTest(options)` | A `Database` and a `TestDatabase`, with no requirements | +| `TestDatabase` | The control side: `statements`, `reply(...replies)`, `reset` | +| `TestDatabaseOptions` | `dialect` (required), `respond`, `requireTenantScope` | +| `TestReply` | Rows as the server sends them, or a `SqlError` | +| `TestStatement` | `{ sql, parameters }`, as received | + +The fake does not run SQL. A test that needs real SQL semantics (constraints, joins over stored +rows) needs a real database: PGlite for Postgres, as in [Database](./database.md#a-complete-example). +A COMMIT or ROLLBACK that fails cannot be scripted yet. diff --git a/scripts/check-doc-examples.mjs b/scripts/check-doc-examples.mjs index d8b190a..de4bef0 100644 --- a/scripts/check-doc-examples.mjs +++ b/scripts/check-doc-examples.mjs @@ -119,6 +119,8 @@ await assert.rejects(() => Effect.runPromise(enums.shared.decodeRows([{ id: "c", assert.match(sql(enums.upsert), /ON CONFLICT \\("slug"\\) DO UPDATE SET/) const database = await import("./database-transaction") assert.deepEqual(database.balances, { from: 70, to: 30, refused: "InsufficientFunds" }) +const testLayer = await import("./database-test-layer") +assert.deepEqual(testLayer.result, { rows: [{ id: 1, balance: 70 }], parameters: ["org_1"] }) console.log("Markdown example behavior checks passed") `, ) diff --git a/src/database.ts b/src/database.ts index ba66164..23e40c5 100644 --- a/src/database.ts +++ b/src/database.ts @@ -3,7 +3,8 @@ // Runs compiled statements through a `SqlClient` you provide, and wraps them in // transactions: Effect's own `withTransaction` plus isolation settings, typed // COMMIT and ROLLBACK failures, contention retry, and `requireTransaction` for -// helpers that must be atomic. See docs/database.md. +// helpers that must be atomic. See docs/database.md. `layerTest` is a +// `Database` with no server, for unit tests; see docs/testing.md. export { Database, @@ -39,5 +40,13 @@ export { type TransactionError, } from "./database/errors" export { QuerySummary, withQuerySummary } from "./database/tracing" +export { + TestDatabase, + layerTest, + type TestDatabaseApi, + type TestDatabaseOptions, + type TestReply, + type TestStatement, +} from "./database/test-layer" export { sql, type SqlIdentifier, type SqlRawText, type SqlTemplate } from "./database/sql" export type { DialectTransactions, IsolationLevel, TransactionSettings } from "./ch/dialect" diff --git a/src/database/test-layer.test-d.ts b/src/database/test-layer.test-d.ts new file mode 100644 index 0000000..ef2bccf --- /dev/null +++ b/src/database/test-layer.test-d.ts @@ -0,0 +1,24 @@ +// Type-level tests: `layerTest` needs nothing and gives both services. + +import { Effect, Layer } from "effect" +import { expectTypeOf } from "expect-type" +import * as CH from "../ch/index" +import * as PG from "../postgres" +import * as Db from "../database" + +const Live = Db.layerTest({ dialect: PG.postgresDialect }) +expectTypeOf(Live).toEqualTypeOf>() + +// Rows from `run` keep the query's row type; the layer removes only `Database`. +const Items = CH.table("t", { id: PG.int4, note: PG.text }) +const rows = Db.run(PG.from(Items).select("id", "note")).pipe(Effect.provide(Live)) +expectTypeOf(rows).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ readonly id: number; readonly note: string }>, + Db.DatabaseError | CH.QueryBuilderError | CH.CompiledQueryDecodeError + > +>() + +// A reply is rows or a driver error, nothing else. +// @ts-expect-error -- a bare row is not a reply +Effect.flatMap(Db.TestDatabase.asEffect(), (test) => test.reply({ id: 1 })) diff --git a/src/database/test-layer.test.ts b/src/database/test-layer.test.ts new file mode 100644 index 0000000..bdae218 --- /dev/null +++ b/src/database/test-layer.test.ts @@ -0,0 +1,112 @@ +// `layerTest`: a `Database` with no server. It records each statement it is +// given and answers with the rows the test supplies, so the real compile, +// decode, tenant-scope and transaction code runs above it. + +import { assert, describe, expect, it } from "@effect/vitest" +import { Cause, Effect, Exit } from "effect" +import { SqlError, UniqueViolation } from "effect/sql/SqlError" +import * as CH from "../ch/index" +import * as PG from "../postgres" +import * as Db from "../database" + +const Accounts = PG.table("accounts", { columns: { id: PG.int4, org: PG.text, balance: PG.int8 }, primaryKey: ["id"] }) + +const balanceOf = (org: string) => + Db.run( + PG.from(Accounts) + .select("id", "balance") + .where(($) => [$.org.eq(org)]), + ) + +describe("layerTest", () => { + it.effect("run sends the compiled statement and decodes the given rows with the query's row schema", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + yield* test.reply([{ id: 1, balance: "42" }]) + expect(yield* balanceOf("org_1")).toEqual([{ id: 1, balance: 42 }]) + const [sent] = yield* test.statements + expect(sent?.parameters).toEqual(["org_1"]) + expect(sent?.sql).toContain('"accounts"."org" = $1') + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), + ) + it.effect("a given row that does not match the row schema fails as it would from a server", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + yield* test.reply([{ id: 1, balance: "not a number" }]) + const error = yield* Effect.flip(balanceOf("org_1")) + expect(error).toBeInstanceOf(CH.CompiledQueryDecodeError) + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), + ) + it.effect("a transaction records BEGIN and COMMIT, or ROLLBACK, and they take no reply", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + yield* test.reply([{ id: 1, balance: "5" }]) + expect(yield* Db.transaction(balanceOf("org_1"))).toEqual([{ id: 1, balance: 5 }]) + class Refused { + readonly _tag = "Refused" + } + const refused = yield* Effect.flip( + Db.transaction(Effect.andThen(Db.execute(Db.sql`UPDATE accounts SET balance = 0`), Effect.fail(new Refused()))), + ) + expect(refused).toBeInstanceOf(Refused) + const sent = (yield* test.statements).map((statement) => statement.sql.split(/\s/)[0]) + expect(sent).toEqual(["BEGIN", "SELECT", "COMMIT", "BEGIN", "UPDATE", "ROLLBACK"]) + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), + ) + it.effect("a given driver error becomes the DatabaseError a real driver's would", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + const duplicate = new SqlError({ + reason: new UniqueViolation({ cause: { code: "23505" }, message: "duplicate key value", constraint: "accounts_pkey" }), + }) + yield* test.reply(duplicate) + const error = yield* Effect.flip(Db.execute(Db.sql`INSERT INTO accounts VALUES (${1}, ${"org_1"}, ${0})`)) + expect(error).toBeInstanceOf(Db.DatabaseError) + expect(error.reason).toBe("UniqueViolation") + expect(error.sqlState).toBe("23505") + expect(yield* test.statements).toEqual([{ sql: "INSERT INTO accounts VALUES ($1, $2, $3)", parameters: [1, "org_1", 0] }]) + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), + ) + + it.effect("respond answers every statement that has no queued reply", () => + Effect.gen(function* () { + const respond = (statement: Db.TestStatement) => (statement.parameters[0] === "org_1" ? [{ id: 1, balance: "7" }] : []) + const rows = yield* Effect.all([balanceOf("org_1"), balanceOf("org_2")]).pipe( + Effect.provide(Db.layerTest({ dialect: PG.postgresDialect, respond })), + ) + expect(rows).toEqual([[{ id: 1, balance: 7 }], []]) + }), + ) + it.effect("requireTenantScope refuses an unscoped query before anything is sent", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + const Tenanted = PG.table("accounts", { columns: { id: PG.int4, org: PG.text }, primaryKey: ["id"], tenantColumn: "org" }) + const exit = yield* Effect.exit(Db.run(PG.from(Tenanted).select("id"))) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + expect(yield* Db.run(PG.from(Tenanted).select("id").where(($) => [$.org.eq("org_1")]))).toEqual([]) + expect(yield* test.statements).toHaveLength(1) + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect, requireTenantScope: true }))), + ) + + it.effect("on ClickHouse a transaction fails with TransactionUnsupported and sends nothing", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + const error = yield* Effect.flip(Db.transaction(Db.execute(Db.sql`SELECT 1`))) + expect(error).toBeInstanceOf(Db.TransactionUnsupported) + expect(yield* test.statements).toEqual([]) + }).pipe(Effect.provide(Db.layerTest({ dialect: CH.clickhouseDialect }))), + ) + + it.effect("reset forgets the recorded statements and the queued replies", () => + Effect.gen(function* () { + const test = yield* Db.TestDatabase + yield* test.reply([{ id: 1, balance: "1" }]) + yield* Db.execute(Db.sql`SELECT 1`) + yield* test.reply([{ id: 2, balance: "2" }]) + yield* test.reset + expect(yield* test.statements).toEqual([]) + expect(yield* balanceOf("org_1")).toEqual([]) + }).pipe(Effect.provide(Db.layerTest({ dialect: PG.postgresDialect }))), + ) +}) diff --git a/src/database/test-layer.ts b/src/database/test-layer.ts new file mode 100644 index 0000000..85a8abe --- /dev/null +++ b/src/database/test-layer.ts @@ -0,0 +1,140 @@ +// A `Database` for unit tests: no server, no module mocks. +// +// It is the real `fromSqlClient` over a fake `SqlClient`. The fake records +// each statement and answers with rows the test gives, so compiling, row +// decoding, tenant-scope checks and transaction handling all run as they do +// against a database. Only the server is replaced. See docs/testing.md. + +import { Context, Effect, Layer, Ref, Stream } from "effect" +import * as Reactivity from "effect/reactivity/Reactivity" +import * as SqlClient from "effect/sql/SqlClient" +import type { Connection } from "effect/sql/SqlConnection" +import type { SqlError } from "effect/sql/SqlError" +import * as SqlStatement from "effect/sql/Statement" +import type { Dialect } from "../ch/dialect" +import { Database, fromSqlClient } from "./database" + +/** A statement as the fake database received it. */ +export interface TestStatement { + readonly sql: string + readonly parameters: ReadonlyArray +} + +/** + * The answer to one statement: rows as the server would send them (before + * decoding), or the driver error it would fail with. + */ +export type TestReply = ReadonlyArray> | SqlError + +export interface TestDatabaseOptions { + /** The dialect to compile for: `postgresDialect` or `clickhouseDialect`. */ + readonly dialect: Dialect + /** The answer when no reply is queued. Default: no rows. */ + readonly respond?: (statement: TestStatement) => TestReply + /** As on `fromSqlClient`. */ + readonly requireTenantScope?: boolean +} + +export interface TestDatabaseApi { + /** + * Every statement received, in order. Transaction control (`BEGIN`, + * `COMMIT`, `ROLLBACK`, savepoints) is included. + */ + readonly statements: Effect.Effect> + /** Queue answers for the next statements, one each, in order. Transaction control takes none. */ + readonly reply: (...replies: ReadonlyArray) => Effect.Effect + /** Forget the recorded statements and the queued replies. */ + readonly reset: Effect.Effect +} + +/** The control side of `layerTest`: what was sent, and what to answer. */ +export class TestDatabase extends Context.Service()("@maple-dev/effect-orm/TestDatabase") {} + +const isRows = (reply: TestReply): reply is ReadonlyArray> => Array.isArray(reply) + +// Placeholders are already in the SQL `Database` sends; this compiler only has +// to pass that SQL and its parameters through. +const compiler = SqlStatement.makeCompiler({ + dialect: "pg", + placeholder: (i) => `$${i}`, + onIdentifier: SqlStatement.defaultEscape('"'), + onRecordUpdate: () => ["", []], + onCustom: () => ["", []], +}) + +/** The fake database and its control side, without a layer. */ +const makeTest = ( + options: TestDatabaseOptions, +): Effect.Effect<{ readonly database: Database["Service"]; readonly test: TestDatabaseApi }> => + Effect.gen(function* () { + const sent = yield* Ref.make>([]) + const queued = yield* Ref.make>([]) + const respond = options.respond ?? (() => []) + + const record = (sql: string, parameters: ReadonlyArray) => { + const statement: TestStatement = { sql, parameters: [...parameters] } + return Effect.as(Ref.update(sent, (all) => [...all, statement]), statement) + } + + const answer = (sql: string, parameters: ReadonlyArray): Effect.Effect, SqlError> => + Effect.flatMap(record(sql, parameters), (statement) => + Effect.flatMap( + Ref.modify(queued, (replies): readonly [TestReply | undefined, ReadonlyArray] => [ + replies[0], + replies.slice(1), + ]), + (next) => { + const reply = next ?? respond(statement) + return isRows(reply) ? Effect.succeed(reply) : Effect.fail(reply) + }, + ), + ) + + // Transaction control runs unprepared; it is recorded and always succeeds. + const control = (sql: string, parameters: ReadonlyArray) => Effect.as(record(sql, parameters), []) + + const values = (sql: string, parameters: ReadonlyArray) => + Effect.map(answer(sql, parameters), (rows) => rows.map((row) => Object.values(row))) + + const connection: Connection = { + execute: (sql, parameters, transformRows) => + transformRows ? Effect.map(answer(sql, parameters), transformRows) : answer(sql, parameters), + executeRaw: answer, + executeValues: values, + executeValuesUnprepared: values, + executeUnprepared: control, + executeStream: (sql, parameters, transformRows) => + Stream.fromIterableEffect( + transformRows ? Effect.map(answer(sql, parameters), transformRows) : answer(sql, parameters), + ), + } + + const client = yield* SqlClient.make({ + acquirer: Effect.succeed(connection), + compiler, + // `Database` names the system on its own span. + spanAttributes: [], + }).pipe(Effect.provide(Reactivity.layer)) + + const database = fromSqlClient(client, { + dialect: options.dialect, + ...(options.requireTenantScope === undefined ? undefined : { requireTenantScope: options.requireTenantScope }), + }) + + const test: TestDatabaseApi = { + statements: Ref.get(sent), + reply: (...replies) => Ref.update(queued, (all) => [...all, ...replies]), + reset: Effect.andThen(Ref.set(sent, []), Ref.set(queued, [])), + } + return { database, test } + }) + +/** + * A `Database` with no server, for unit tests, and the `TestDatabase` that + * controls it. Each statement is recorded and answered with the next queued + * reply, or with `respond`. Rows are decoded by the query's own row schema. + */ +export const layerTest = (options: TestDatabaseOptions): Layer.Layer => + Layer.effectContext( + Effect.map(makeTest(options), ({ database, test }) => Context.make(Database, database).pipe(Context.add(TestDatabase, test))), + )