diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index da0f64998..e3528d1c4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,6 +94,7 @@ jobs: # latest Postgres version: - go-version: "1.26" postgres-version: 18 + fail-fast: false timeout-minutes: 5 @@ -139,6 +140,29 @@ jobs: if: ${{ !matrix.primary }} run: make test TEST_DATABASE=postgres + build_and_test_mysql: + env: + GOTOOLCHAIN: local + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version: "1.27" + + # MySQL is pre-installed on GitHub-hosted Ubuntu runners. + - name: Start MySQL + run: | + sudo systemctl start mysql + mysql -u root -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED BY ''; FLUSH PRIVILEGES;" + + - name: Test with race detector + run: make test/race TEST_DATABASE=mysql + build_and_test_sqlite: name: build_and_test_sqlite (${{ matrix.go-version }}${{ matrix.primary && ', race' || '' }}) env: diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml index 4f55e6def..c0e8122ef 100644 --- a/.github/workflows/conformance.yaml +++ b/.github/workflows/conformance.yaml @@ -42,9 +42,27 @@ env: jobs: port_fixtures: - name: Port fixture tests + name: Port fixture tests (SQLite and MySQL) runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 + + env: + RIVER_RUST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + RIVER_TEST_DATABASE_URL: mysql://root@127.0.0.1:3306/ + TEST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 steps: - uses: actions/checkout@v7 @@ -75,16 +93,32 @@ jobs: - name: Rust fixture tests run: make test/rust/conformance + - name: Rust MySQL fixture tests + run: make test/rust/conformance/mysql + - name: JavaScript fixture tests run: make test/js/conformance + - name: JavaScript MySQL fixture tests + run: make test/js/conformance/mysql + - name: Java fixture tests run: make test/java/conformance + - name: Java MySQL fixture tests + run: make test/java/conformance/mysql + - uses: ruby/setup-ruby@v1 with: working-directory: ruby bundler-cache: true - name: Ruby fixture tests + env: + TEST_DATABASE: sqlite run: make test/ruby/conformance + + - name: Ruby MySQL fixture tests + env: + TEST_MYSQL_URL: mysql2://root@127.0.0.1:3306/ + run: make test/ruby/conformance/mysql diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 007f59dc1..3bec2929d 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -11,6 +11,7 @@ on: - "Makefile" - "conformance/**" - "java/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" pull_request: @@ -20,6 +21,7 @@ on: - "Makefile" - "conformance/**" - "java/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" workflow_dispatch: @@ -130,3 +132,35 @@ jobs: - name: Postgres client, worker, and migration tests run: make test/java/postgres + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_TEST_DATABASE_URL: mysql://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ./.github/actions/setup-java + - uses: actions/setup-go@v6 + with: + go-version-file: go.work + - run: make test/java/mysql diff --git a/.github/workflows/js.yaml b/.github/workflows/js.yaml index dab55a710..9bcc3932e 100644 --- a/.github/workflows/js.yaml +++ b/.github/workflows/js.yaml @@ -250,3 +250,36 @@ jobs: - run: node cli/dist/bin.js migrate-up --database-url "$TEST_DATABASE_URL" - run: pnpm run test:integration + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + TEST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ./.github/actions/setup-js + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + - run: make test/js/mysql + working-directory: . diff --git a/.github/workflows/ruby.yaml b/.github/workflows/ruby.yaml index 794389382..d1088e435 100644 --- a/.github/workflows/ruby.yaml +++ b/.github/workflows/ruby.yaml @@ -33,8 +33,8 @@ defaults: jobs: checks: - # Check coverage from the two backends without adding another CI job. - needs: [test, test_sqlite] + # Check coverage from all backends without adding another CI job. + needs: [test, test_sqlite, mysql] runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -207,3 +207,49 @@ jobs: env: RAILS_VERSION: "~> 8.1.0" run: bundle install && bundle exec rspec + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_REQUIRE_DATABASES: "1" + TEST_DATABASE: mysql + TEST_MYSQL_URL: mysql2://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "4.0" + working-directory: ruby + bundler-cache: true + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + - run: make generate/fixtures spec/core + - uses: actions/upload-artifact@v7 + if: matrix.mysql-version == '8.4' + with: + name: ruby-coverage-mysql + path: ruby/coverage/.resultset.json + include-hidden-files: true + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 2bdc7278a..200281ea6 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -11,6 +11,7 @@ on: - "Makefile" - "conformance/**" - "internal/cmd/syncrustmigrations/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" - "rust/**" @@ -20,6 +21,7 @@ on: - "Makefile" - "conformance/**" - "internal/cmd/syncrustmigrations/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" - "rust/**" @@ -53,7 +55,7 @@ jobs: with: tool: cargo-deny@0.20.2 - - name: Lint, including Postgres-only and SQLite-only builds + - name: Lint, including each backend on its own run: make lint/rust - name: Documentation and examples @@ -191,3 +193,34 @@ jobs: run: PGPASSWORD=postgres createdb -h localhost -U postgres river_rust_test - name: Rust unit, SQLite, and Postgres tests run: make test/rust + + mysql: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_RUST_MYSQL_URL: mysql://root:root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -proot" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy,rustfmt + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + - run: make test/rust/mysql diff --git a/Makefile b/Makefile index 4c9f13ab6..f34481a5e 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ generate/rust-migrations: ## Sync database migrations to Rust .PHONY: generate/sqlc generate/sqlc: ## Generate sqlc cd riverdriver/riverdatabasesql/internal/dbsqlc && $(SQLC) generate + cd riverdriver/rivermysql/internal/dbsqlc && $(SQLC) generate cd riverdriver/riverpgxv5/internal/dbsqlc && $(SQLC) generate cd riverdriver/riversqlite/internal/dbsqlc && $(SQLC) generate @@ -79,19 +80,25 @@ TEST_DATABASE ?= all # Only filter the shared driver suite. Other packages have SQLite-named tests # that use Postgres or mocks and should stay in the regular test run. +mysql_test_pattern := ^(Test.*MySQL|Example_mysql) sqlite_test_pattern := ^(Test.*(LibSQL|SQLite|Turso)|Example_(libSQL|sqlite|turso)) test_submodules := $(submodules) legacy_driver_test_flags := -run '/WithTx$$' -ifeq ($(TEST_DATABASE),postgres) - test_submodules := $(filter-out %/riverdriver/riversqlite,$(submodules)) - driver_test_flags := -skip '$(sqlite_test_pattern)' - legacy_driver_test_flags += -skip '$(sqlite_test_pattern)' +ifeq ($(TEST_DATABASE),mysql) + export RIVER_MYSQL_TESTS_ENABLED := 1 + test_submodules := $(filter %/riverdriver/riverdrivertest %/riverdriver/rivermysql,$(submodules)) + driver_test_flags := -run '$(mysql_test_pattern)' + legacy_driver_test_flags := -run '$(mysql_test_pattern)/WithTx$$' +else ifeq ($(TEST_DATABASE),postgres) + test_submodules := $(filter-out %/riverdriver/rivermysql %/riverdriver/riversqlite,$(submodules)) + driver_test_flags := -skip '$(mysql_test_pattern)|$(sqlite_test_pattern)' + legacy_driver_test_flags += -skip '$(mysql_test_pattern)|$(sqlite_test_pattern)' else ifeq ($(TEST_DATABASE),sqlite) test_submodules := $(filter %/riverdriver/riverdrivertest %/riverdriver/riversqlite,$(submodules)) driver_test_flags := -run '$(sqlite_test_pattern)' legacy_driver_test_flags := -run '$(sqlite_test_pattern)/WithTx$$' else ifneq ($(TEST_DATABASE),all) - $(error TEST_DATABASE must be all, postgres, or sqlite) + $(error TEST_DATABASE must be all, mysql, postgres, or sqlite) endif # Definitions of following tasks look ugly, but they're done this way because to @@ -111,6 +118,9 @@ lint/rust: ## Run Rust formatting and clippy checks, including single-backend bu cd rust && cargo fmt --all -- --check cd rust && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features postgres --all-targets --locked -- -D warnings + cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features mysql --all-targets --locked -- -D warnings + # Test dependencies enable other backends; also check MySQL libraries on their own. + cd rust && cargo clippy -p riverqueue -p riverqueue-migrate --lib --no-default-features --features mysql --locked -- -D warnings cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features sqlite --all-targets --locked -- -D warnings cd rust && $(RUST_POSTGRES_TESTS_ENV) cargo clippy -p riverqueue -p riverqueue-migrate --all-targets --all-features --locked -- -D warnings @@ -147,7 +157,7 @@ lint/js: build/js pnpm -C js run typecheck:next .PHONY: test -test:: ## Run tests (TEST_DATABASE=all, postgres, or sqlite) +test:: ## Run tests (TEST_DATABASE=all, mysql, postgres, or sqlite) define test-target test:: ; cd $1 && go test ./... -timeout 2m $(if $(filter %/riverdriver/riverdrivertest,$1),$(driver_test_flags)) endef @@ -155,7 +165,7 @@ $(foreach mod,$(test_submodules),$(eval $(call test-target,$(mod)))) # Exercise the temporary savepoint fallback as well as default transaction reuse. test:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . $(legacy_driver_test_flags) -timeout 2m -ifneq ($(TEST_DATABASE),sqlite) +ifneq ($(filter all postgres,$(TEST_DATABASE)),) test:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -run '^TestDriverRiverPgxV5$$/.*/WithTx$$' -timeout 2m endif @@ -180,6 +190,16 @@ test/java/conformance: ## Run Java tests that check Go-generated conformance fix test/java/conformance: generate/fixtures $(MVN) --batch-mode --no-transfer-progress -f java/pom.xml -pl river test -Dgroups=conformance -Driver.test.database=sqlite +.PHONY: test/java/conformance/mysql +test/java/conformance/mysql: generate/fixtures ## Run Java's Go fixture checks on MySQL (requires RIVER_TEST_DATABASE_URL) + @test -n "$$RIVER_TEST_DATABASE_URL" || { echo "RIVER_TEST_DATABASE_URL is required" >&2; exit 1; } + $(MVN) --batch-mode --no-transfer-progress -f java/pom.xml -pl river test -Dgroups=conformance -Driver.test.database=mysql + +.PHONY: test/java/mysql +test/java/mysql: generate/fixtures verify/java-migrations ## Run Java tests against MySQL (requires RIVER_TEST_DATABASE_URL) + @test -n "$$RIVER_TEST_DATABASE_URL" || { echo "RIVER_TEST_DATABASE_URL is required" >&2; exit 1; } + $(MVN) --batch-mode --no-transfer-progress -f java/pom.xml verify -Driver.test.database=mysql -DexcludedGroups=postgres + .PHONY: test/java/postgres test/java/postgres: ## Run Java tests with Postgres client and worker coverage (requires RIVER_TEST_DATABASE_URL) test/java/postgres: generate/fixtures verify/java-migrations @@ -213,9 +233,19 @@ test/js/conformance: ## Run JavaScript tests that check Go-generated conformance test/js/conformance: generate/fixtures pnpm -C js exec vitest run src/conformance.test.ts src/cron.test.ts src/runtime/completion-command.test.ts src/runtime/notification-pump.conformance.test.ts +.PHONY: test/js/conformance/mysql +test/js/conformance/mysql: generate/fixtures ## Run JavaScript's Go fixture checks on MySQL (requires TEST_MYSQL_URL) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + pnpm -C js exec vitest run --config vitest.integration.config.ts driver/mysql/src/driver.integration.test.ts -t 'Go conformance' + # Integration tests use TEST_DATABASE_URL (default # postgres://localhost:5432/river_test), migrated with # `node js/cli/dist/bin.js migrate-up`. +.PHONY: test/js/mysql +test/js/mysql: build/js generate/fixtures ## Run JavaScript MySQL integration tests (requires TEST_MYSQL_URL) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + pnpm -C js exec vitest run --config vitest.integration.config.ts driver/mysql/src/driver.integration.test.ts + .PHONY: test/js/integration test/js/integration: ## Run JavaScript integration tests against Postgres test/js/integration: build/js @@ -240,6 +270,17 @@ test/rust/conformance: ## Run Rust tests that check Go-generated conformance fix test/rust/conformance: generate/fixtures cd rust && cargo test -p riverqueue --features chrono-tz --lib --test protocol_fixtures --locked +.PHONY: test/rust/conformance/mysql +test/rust/conformance/mysql: generate/fixtures ## Run Rust's Go fixture checks on MySQL (requires RIVER_RUST_MYSQL_URL) + @test -n "$$RIVER_RUST_MYSQL_URL" || { echo "RIVER_RUST_MYSQL_URL is required" >&2; exit 1; } + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo test -p riverqueue --all-features --lib database::mysql::tests::conformance --locked + +.PHONY: test/rust/mysql +test/rust/mysql: generate/fixtures verify/rust-migrations ## Run Rust MySQL integration tests (requires RIVER_RUST_MYSQL_URL) + @test -n "$$RIVER_RUST_MYSQL_URL" || { echo "RIVER_RUST_MYSQL_URL is required" >&2; exit 1; } + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo test -p riverqueue --all-features --lib database::mysql::tests --locked + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo clippy -p riverqueue --all-features --tests --locked -- -D warnings + .PHONY: test/rust/postgres test/rust/postgres: ## Run all Rust tests, including Postgres integration tests (requires RIVER_RUST_DATABASE_URL) test/rust/postgres: generate/fixtures @@ -263,6 +304,7 @@ doc/rust: ## Build Rust API documentation, compiled examples, and doctests for e cd rust && RUSTDOCFLAGS="-D warnings" cargo doc --workspace --all-features --no-deps --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test --workspace --all-features --doc --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features postgres --doc --locked + cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features mysql --doc --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features sqlite --doc --locked cd rust && cargo check --workspace --examples --all-features --locked @@ -322,14 +364,14 @@ check/rust/semver: ## Check Rust APIs against RUST_SEMVER_BASELINE_REV (default: fi .PHONY: test/race -test/race:: ## Run tests with race detector (TEST_DATABASE=all, postgres, or sqlite) +test/race:: ## Run tests with race detector (TEST_DATABASE=all, mysql, postgres, or sqlite) define test-race-target test/race:: ; cd $1 && go test ./... -race -timeout 2m $(if $(filter %/riverdriver/riverdrivertest,$1),$(driver_test_flags)) endef $(foreach mod,$(test_submodules),$(eval $(call test-race-target,$(mod)))) test/race:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -race $(legacy_driver_test_flags) -timeout 2m -ifneq ($(TEST_DATABASE),sqlite) +ifneq ($(filter all postgres,$(TEST_DATABASE)),) test/race:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -race -run '^TestDriverRiverPgxV5$$/.*/WithTx$$' -timeout 2m endif @@ -387,6 +429,7 @@ verify/rust-migrations: ## Verify Rust migrations match the canonical migrations .PHONY: verify/sqlc verify/sqlc: ## Verify generated sqlc cd riverdriver/riverdatabasesql/internal/dbsqlc && $(SQLC) diff + cd riverdriver/rivermysql/internal/dbsqlc && $(SQLC) diff cd riverdriver/riverpgxv5/internal/dbsqlc && $(SQLC) diff cd riverdriver/riversqlite/internal/dbsqlc && $(SQLC) diff @@ -405,6 +448,11 @@ test/ruby: ## Run Ruby core, Active Record, Sequel, and Rails tests test/ruby/conformance: ## Run Ruby tests that check Go-generated conformance fixtures $(MAKE) -C ruby test/conformance +.PHONY: test/ruby/conformance/mysql +test/ruby/conformance/mysql: ## Run Ruby's Go fixture checks on MySQL (requires TEST_MYSQL_URL with mysql2://) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + TEST_DATABASE=mysql $(MAKE) -C ruby test/conformance + typecheck/ruby: ## Check Ruby types with Steep $(MAKE) -C ruby type-check diff --git a/client.go b/client.go index e06af4b0c..5deebb100 100644 --- a/client.go +++ b/client.go @@ -1067,11 +1067,11 @@ func NewClient[TTx any](driver riverdriver.Driver[TTx], config *Config) (*Client client.testSignals.queueCleaner = &queueCleaner.TestSignals } - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - sqliteNotificationCleaner := maintenance.NewSQLiteNotificationCleaner(archetype, &maintenance.SQLiteNotificationCleanerConfig{ + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + notificationCleaner := maintenance.NewSQLiteNotificationCleaner(archetype, &maintenance.SQLiteNotificationCleanerConfig{ Schema: config.Schema, }, driver.GetExecutor()) - maintenanceServices = append(maintenanceServices, sqliteNotificationCleaner) + maintenanceServices = append(maintenanceServices, notificationCleaner) } { @@ -2543,11 +2543,17 @@ func (c *Client[TTx]) JobList(ctx context.Context, params *JobListParams) (*JobL if params == nil { params = NewJobListParams() } - params.schema = c.config.Schema - if c.driver.DatabaseName() == riverdriver.DatabaseNameSQLite && params.metadataCalled { return nil, errJobListParamsMetadataNotSupportedSQLite } + if params.metadataCalled { + var err error + params, err = params.withMetadataPredicateSQL(c.driver.SQLFragmentColumnContainsJSON) + if err != nil { + return nil, err + } + } + params.schema = c.config.Schema dbParams, err := params.toDBParams() if err != nil { @@ -2584,11 +2590,17 @@ func (c *Client[TTx]) JobListTx(ctx context.Context, tx TTx, params *JobListPara if params == nil { params = NewJobListParams() } - params.schema = c.config.Schema - if c.driver.DatabaseName() == riverdriver.DatabaseNameSQLite && params.metadataCalled { return nil, errJobListParamsMetadataNotSupportedSQLite } + if params.metadataCalled { + var err error + params, err = params.withMetadataPredicateSQL(c.driver.SQLFragmentColumnContainsJSON) + if err != nil { + return nil, err + } + } + params.schema = c.config.Schema dbParams, err := params.toDBParams() if err != nil { diff --git a/client_test.go b/client_test.go index 472bac298..0a5f5cef6 100644 --- a/client_test.go +++ b/client_test.go @@ -8956,6 +8956,32 @@ func Test_NewClient_Overrides(t *testing.T) { require.Len(t, client.config.WorkerMiddleware, 1) } +func Test_NewClient_MySQLNotificationCleaner(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + var ( + dbPool = riversharedtest.DBPool(ctx, t) + pgxDriver = riverpgxv5.New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, pgxDriver, nil) + ) + + workers := NewWorkers() + AddWorker(workers, &noOpWorker{}) + + client, err := NewClient(NewDriverMySQLDatabaseName(dbPool), &Config{ + Queues: map[string]QueueConfig{QueueDefault: {MaxWorkers: 1}}, + Schema: schema, + TestOnly: true, + Workers: workers, + }) + require.NoError(t, err) + + notificationCleaner := maintenance.GetService[*maintenance.SQLiteNotificationCleaner](client.queueMaintainer) + require.Equal(t, schema, notificationCleaner.Config.Schema) +} + func Test_NewClient_PluginsAndHybrids(t *testing.T) { t.Parallel() @@ -10487,6 +10513,23 @@ func TestDefaultClientIDWithHost(t *testing.T) { require.Equal(t, strings.Repeat("a", 60)+"_2024_03_07T04_39_12_123456", defaultClientIDWithHost(startedAt, strings.Repeat("a", 61))) } +// DriverMySQLDatabaseName is a pgx driver that reports MySQL as its database +// name so MySQL-specific client wiring can be tested without importing the +// separate MySQL driver module. +type DriverMySQLDatabaseName struct { + riverpgxv5.Driver +} + +// NewDriverMySQLDatabaseName returns a new test driver that reports MySQL as +// its database name. +func NewDriverMySQLDatabaseName(dbPool *pgxpool.Pool) *DriverMySQLDatabaseName { + return &DriverMySQLDatabaseName{ + Driver: *riverpgxv5.New(dbPool), + } +} + +func (d *DriverMySQLDatabaseName) DatabaseName() string { return riverdriver.DatabaseNameMySQL } + // DriverPollOnly simulates a driver without a listener. An example of this is // Postgres through `riverdatabasesql`, which is Postgres (so it can notify), // but where `database/sql` provides no listener mechanism. We could use the diff --git a/docs/development.md b/docs/development.md index 8b0454959..5994a919b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,21 +13,30 @@ Run tests: make test -By default, this runs all workspace modules and database drivers. Use -`make test/race` to enable the race detector. +By default, this runs all workspace modules and database drivers except MySQL, +which is opt-in. Use `make test/race` to enable the race detector. + +To run only the MySQL driver tests (no Postgres needed): + + make test TEST_DATABASE=mysql + +These connect as `root` without a password on `localhost:3306`. Set +`TEST_MYSQL_URL` to override the connection DSN. To include MySQL in the full +suite, set `RIVER_MYSQL_TESTS_ENABLED=1`. To run only the SQLite, libSQL, and Turso driver tests (no Postgres needed): make test TEST_DATABASE=sqlite -To run the regular suite with the SQLite driver tests excluded: +To run the regular suite with the MySQL and SQLite driver tests excluded: make test TEST_DATABASE=postgres `TEST_DATABASE` also applies to `make test/race`. CI separates these suites so -the Postgres version matrix doesn't repeat the SQLite tests. Each suite still -runs on both supported Go versions. Only the latest combination in each matrix -uses the race detector, marked `race` in its job title. +the Postgres version matrix doesn't repeat the MySQL and SQLite tests. Postgres +and SQLite run on both supported Go versions, with the latest combination in +each matrix using the race detector. MySQL runs separately with the race +detector on the latest supported Go version. ## Run lint diff --git a/go.work b/go.work index 7f9949533..905844064 100644 --- a/go.work +++ b/go.work @@ -10,6 +10,7 @@ use ( ./riverdriver/riverdatabasesql ./riverdriver/riverdrivertest ./riverdriver/riverpgxv5 + ./riverdriver/rivermysql ./riverdriver/riversqlite ./rivershared ./rivertype diff --git a/internal/cmd/syncrustmigrations/main.go b/internal/cmd/syncrustmigrations/main.go index 576738863..7c64b531a 100644 --- a/internal/cmd/syncrustmigrations/main.go +++ b/internal/cmd/syncrustmigrations/main.go @@ -22,6 +22,10 @@ func main() { flag.Parse() databases := []database{ + { + canonicalDir: "riverdriver/rivermysql/migration/main", + mirrorDir: "rust/riverqueue-migrate/migrations/mysql/main", + }, { canonicalDir: "riverdriver/riverpgxv5/migration/main", mirrorDir: "rust/riverqueue-migrate/migrations/main", diff --git a/internal/dblist/db_list.go b/internal/dblist/db_list.go index b91596dee..1e1febc81 100644 --- a/internal/dblist/db_list.go +++ b/internal/dblist/db_list.go @@ -44,6 +44,7 @@ type WherePredicate struct { } type sqlFragmentBuilder interface { + DatabaseName() string SQLFragmentColumnContainsAll(column, namedArg string, values []string) (string, any, error) SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) SQLFragmentColumnIn(column string, values any) (string, any, error) @@ -197,6 +198,17 @@ func JobMakeDriverParams(ctx context.Context, params *JobListParams, sqlFragment var orderByBuilder strings.Builder for i, orderBy := range params.OrderBy { + // MySQL doesn't support NULLS FIRST/LAST. Sort by nullness first to + // preserve the same cursor ordering as Postgres and SQLite. + mysqlNullOrder := sqlFragmentBuilder.DatabaseName() == riverdriver.DatabaseNameMySQL && orderBy.Expr != "id" + if mysqlNullOrder { + orderByBuilder.WriteString(orderBy.Expr) + if orderBy.Order == SortOrderAsc { + orderByBuilder.WriteString(" IS NULL ASC, ") + } else { + orderByBuilder.WriteString(" IS NULL DESC, ") + } + } orderByBuilder.WriteString(orderBy.Expr) // Match Postgres's default null placement on every driver. Leave the // non-null ID tie-breaker alone: SQLite otherwise adds a temporary sort @@ -204,12 +216,12 @@ func JobMakeDriverParams(ctx context.Context, params *JobListParams, sqlFragment switch orderBy.Order { case SortOrderAsc: orderByBuilder.WriteString(" ASC") - if orderBy.Expr != "id" { + if orderBy.Expr != "id" && !mysqlNullOrder { orderByBuilder.WriteString(" NULLS LAST") } case SortOrderDesc: orderByBuilder.WriteString(" DESC") - if orderBy.Expr != "id" { + if orderBy.Expr != "id" && !mysqlNullOrder { orderByBuilder.WriteString(" NULLS FIRST") } case SortOrderUnspecified: diff --git a/internal/maintenance/job_rescuer_test.go b/internal/maintenance/job_rescuer_test.go index b5d40f892..1903295b7 100644 --- a/internal/maintenance/job_rescuer_test.go +++ b/internal/maintenance/job_rescuer_test.go @@ -7,6 +7,7 @@ import ( "math" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/require" @@ -77,6 +78,14 @@ func (p *SimpleClientRetryPolicy) NextRetry(job *rivertype.JobRow) time.Time { return job.AttemptedAt.Add(timeutil.SecondsAsDuration(retrySeconds)) } +type jobRescuerExecutorMock struct { + riverdriver.Executor +} + +func (e *jobRescuerExecutorMock) JobGetStuck(context.Context, *riverdriver.JobGetStuckParams) ([]*rivertype.JobRow, error) { + return nil, nil +} + type jobRescuerPilotSpy struct { riverpilot.StandardPilot @@ -359,16 +368,38 @@ func TestJobRescuer(t *testing.T) { t.Run("CustomizableInterval", func(t *testing.T) { t.Parallel() - cleaner, _ := setup(t) - cleaner.Config.Interval = 1 * time.Microsecond - - require.NoError(t, cleaner.Start(ctx)) - - // This should trigger ~immediately every time: - for i := range 5 { - t.Logf("Iteration %d", i) - cleaner.TestSignals.FetchedBatch.WaitOrTimeout() - } + synctest.Test(t, func(t *testing.T) { + // Keep database I/O out of the fake-clock test so each interval runs + // only when time advances, without overflowing either test signal. + const interval = time.Minute + rescuer := NewRescuer(riversharedtest.BaseServiceArchetype(t), &JobRescuerConfig{ + ClientRetryPolicy: &SimpleClientRetryPolicy{}, + Interval: interval, + WorkUnitFactoryFunc: func(string) workunit.WorkUnitFactory { + return nil + }, + }, &jobRescuerExecutorMock{}) + rescuer.StaggerStartupDisable(true) + rescuer.TestSignals.Init(t) + t.Cleanup(rescuer.Stop) + + require.NoError(t, rescuer.Start(ctx)) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.WaitOrTimeout() + rescuer.TestSignals.UpdatedBatch.WaitOrTimeout() + + for range 5 { + time.Sleep(interval - time.Nanosecond) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.RequireEmpty() + rescuer.TestSignals.UpdatedBatch.RequireEmpty() + + time.Sleep(time.Nanosecond) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.WaitOrTimeout() + rescuer.TestSignals.UpdatedBatch.WaitOrTimeout() + } + }) }) t.Run("StopsImmediately", func(t *testing.T) { diff --git a/internal/maintenance/sqlite_notification_cleaner.go b/internal/maintenance/sqlite_notification_cleaner.go index f5b6fbb59..62a1b9748 100644 --- a/internal/maintenance/sqlite_notification_cleaner.go +++ b/internal/maintenance/sqlite_notification_cleaner.go @@ -67,9 +67,8 @@ func (c *SQLiteNotificationCleanerConfig) mustValidate() *SQLiteNotificationClea return c } -// SQLiteNotificationCleaner periodically removes old rows from SQLite's -// notification outbox. It is only needed for the SQLite driver's emulated -// listen/notify support. +// SQLiteNotificationCleaner periodically removes old rows from the durable +// notification outbox used to emulate listen/notify for SQLite and MySQL. type SQLiteNotificationCleaner struct { riversharedmaintenance.QueueMaintainerServiceBase startstop.BaseStartStop @@ -84,7 +83,7 @@ type SQLiteNotificationCleaner struct { reducedBatchSizeBreaker *circuitbreaker.CircuitBreaker } -// NewSQLiteNotificationCleaner returns a SQLite notification cleaner. +// NewSQLiteNotificationCleaner returns a notification outbox cleaner. func NewSQLiteNotificationCleaner(archetype *baseservice.Archetype, config *SQLiteNotificationCleanerConfig, exec riverdriver.Executor) *SQLiteNotificationCleaner { batchSizes := config.WithDefaults() @@ -127,7 +126,7 @@ func (s *SQLiteNotificationCleaner) Start(ctx context.Context) error { //nolint: res, err := s.runOnce(ctx) if err != nil { if !errors.Is(err, context.Canceled) { - s.Logger.ErrorContext(ctx, s.Name+": Error cleaning SQLite notifications", slog.String("error", err.Error())) + s.Logger.ErrorContext(ctx, s.Name+": Error cleaning notification outbox", slog.String("error", err.Error())) } continue } diff --git a/java/README.md b/java/README.md index b6b14557c..e03469617 100644 --- a/java/README.md +++ b/java/README.md @@ -1,6 +1,6 @@ # River for Java -Prerelease Java 21 client and worker runtime for River's Postgres and SQLite +Prerelease Java 21 client and worker runtime for River's MySQL, Postgres, and SQLite schemas. Jobs are ordinary River jobs: another language can insert, cancel, retry, or work them using the same database. @@ -21,14 +21,14 @@ Java 21 or newer is required. Add `com.riverqueue:river:0.48.0-alpha.1` and a JD ``` -For SQLite, replace the Postgres JDBC dependency with `org.xerial:sqlite-jdbc:3.53.4.0`. To install a local copy of River from source, see the [development guide](docs/development.md#building-from-source). +For MySQL, use `com.mysql:mysql-connector-j:26.7.0`. For SQLite, replace the Postgres JDBC dependency with `org.xerial:sqlite-jdbc:3.53.4.0`. To install a local copy of River from source, see the [development guide](docs/development.md#building-from-source). ## Quick start Define job arguments as a record, give the job a stable kind, and register a worker lambda. This complete `Example.java` starts a client, inserts a job, waits for its completion, and stops the workers. Set `DATABASE_URL` to a -Postgres URL or a file-backed SQLite JDBC URL. +MySQL or Postgres URL, or a file-backed SQLite JDBC URL. ```java import com.riverqueue.*; @@ -78,7 +78,7 @@ for each operation. River never closes an application-owned data source. An ## Migration CLI -The `river-cli` executable JAR includes Postgres and SQLite JDBC drivers and +The `river-cli` executable JAR includes MySQL, Postgres, and SQLite JDBC drivers and runs with Java 21 or newer. It needs no application classpath. For a local build, see the [development guide](docs/development.md#building-from-source). Set `RIVER_CLI` to the executable JAR, then run: @@ -119,8 +119,8 @@ java -jar "$RIVER_CLI" migrate-get --version 3 --up > river_3.up.sql java -jar "$RIVER_CLI" migrate-get --driver sqlite --all --up > river_sqlite.up.sql ``` -SQL export defaults to Postgres without a database URL. Use `--driver sqlite` -for SQLite, or select the dialect through the URL. Export accepts comma-separated +SQL export defaults to Postgres without a database URL. Use `--driver mysql` or `--driver sqlite` +for MySQL or SQLite, or select the dialect through the URL. Export accepts comma-separated `--version` values or `--all`; `--exclude-version 1` excludes migration history setup when exporting for another migration framework. Exit codes are `0` for success, `1` for an operation failure, and `2` for invalid arguments. @@ -588,6 +588,23 @@ Workers.Handler sendEmail = context -> workerConfig.add(email, sendEmail); ``` +## MySQL + +Use MySQL 8.0 or later with the latest patch release. `Database.connect` accepts +`mysql://user:password@localhost/my_app` and `jdbc:mysql://localhost/my_app`. +Applications with an existing pool can use `new Database(dataSource, Database.Dialect.MYSQL)`. +`withSchema("background_jobs")` selects a separate MySQL database for River's tables. + +Workers claim jobs with `FOR UPDATE SKIP LOCKED` and receive notifications +through the durable outbox, including cancellation and leadership resignation. +River-owned transactions use READ COMMITTED; caller transactions keep their +configured isolation level. Inserts and job completion participate in caller +transactions through the same APIs as Postgres. + +MySQL DDL commits implicitly. Migration history is recorded after each successful +version, but a failed version may leave partially applied DDL. Inspect and repair +it before retrying. A named database lock prevents concurrent migration runners. + ## SQLite Use a [file database](https://riverqueue.com/docs/sqlite) shared by every diff --git a/java/bin/check-packages/main.go b/java/bin/check-packages/main.go index ee1ed072e..7dfbac499 100644 --- a/java/bin/check-packages/main.go +++ b/java/bin/check-packages/main.go @@ -15,7 +15,10 @@ import ( "strings" ) -var forbiddenPath = regexp.MustCompile(`(^|/)(fixtures?|testdata|tests|conformance)/|(?:Test|Tests)(?:\$[^/]*)?\.(class|java)$`) +// Dependencies in the shaded CLI may have production classes ending in Test +// (for example, protobuf's UnredactedDebugFormatForTest). Check that suffix +// only in River's namespace. +var forbiddenPath = regexp.MustCompile(`(^|/)(fixtures?|testdata|tests|conformance)/|^com/riverqueue/.*(?:Test|Tests)(?:\$[^/]*)?\.(class|java)$`) func main() { if err := run("java"); err != nil { diff --git a/java/bin/check-packages/main_test.go b/java/bin/check-packages/main_test.go index 65b76b96e..87ed8bcda 100644 --- a/java/bin/check-packages/main_test.go +++ b/java/bin/check-packages/main_test.go @@ -20,6 +20,7 @@ func TestCheckArchive(t *testing.T) { wantError string }{ {name: "Conformance", extra: "com/riverqueue/conformance/Adapter.class", wantError: "development content"}, + {name: "DependencyProductionClass", extra: "com/google/protobuf/UnredactedDebugFormatForTest.class", resource: "SELECT 1;"}, {name: "Fixture", extra: "fixtures/golden.json", wantError: "development content"}, {name: "InnerTestClass", extra: "com/riverqueue/ClientTest$Helper.class", wantError: "development content"}, {name: "JSON", extra: "com/riverqueue/protocol.json", wantError: "development content"}, diff --git a/java/bin/sync-migrations/main.go b/java/bin/sync-migrations/main.go index 6574130e9..c88d7c01f 100644 --- a/java/bin/sync-migrations/main.go +++ b/java/bin/sync-migrations/main.go @@ -38,6 +38,7 @@ func run(root string, check bool) error { dialect string driver string }{ + {"mysql", "rivermysql"}, {"postgres", "riverpgxv5"}, {"sqlite", "riversqlite"}, } { @@ -68,7 +69,7 @@ func run(root string, check bool) error { } } if names != nil && !slices.Equal(current, names) { - return errors.New("migration names differ between Postgres and SQLite; review the Java catalog") + return errors.New("migration names differ between database backends; review the Java catalog") } names = current } diff --git a/java/bin/sync-migrations/main_test.go b/java/bin/sync-migrations/main_test.go index 6db5a1d7c..867c3f683 100644 --- a/java/bin/sync-migrations/main_test.go +++ b/java/bin/sync-migrations/main_test.go @@ -24,7 +24,7 @@ func TestRun(t *testing.T) { destination: filepath.Join(root, "java/river/src/main/resources/com/riverqueue/migration"), root: root, } - for _, driver := range []string{"riverpgxv5", "riversqlite"} { + for _, driver := range []string{"rivermysql", "riverpgxv5", "riversqlite"} { source := filepath.Join(root, "riverdriver", driver, "migration/main") require.NoError(t, os.MkdirAll(source, 0o755)) for _, direction := range []string{"up", "down"} { @@ -50,7 +50,7 @@ func TestRun(t *testing.T) { missing := filepath.Join(bundle.destination, "postgres/001_first.down.sql") stale := filepath.Join(bundle.destination, "sqlite/001_first.up.sql") - extra := filepath.Join(bundle.destination, "sqlite/999_extra.up.sql") + extra := filepath.Join(bundle.destination, "mysql/999_extra.up.sql") require.NoError(t, os.Remove(missing)) require.NoError(t, os.WriteFile(stale, []byte("stale"), 0o600)) require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600)) diff --git a/java/cli/pom.xml b/java/cli/pom.xml index 76e14a5ea..9e6ec2369 100644 --- a/java/cli/pom.xml +++ b/java/cli/pom.xml @@ -6,6 +6,7 @@ Mozilla Public License 2.0https://mozilla.org/MPL/2.0/repo com.riverqueueriver0.48.0-alpha.1 + com.mysqlmysql-connector-j org.postgresqlpostgresql org.xerialsqlite-jdbc org.junit.jupiterjunit-jupitertest diff --git a/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java b/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java index bdee00485..ffdf18a53 100644 --- a/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java +++ b/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java @@ -33,19 +33,26 @@ private static Database database( Options options, Map environment, boolean offline) { String url = options.values.getOrDefault("database-url", environment.get("DATABASE_URL")); String driver = options.values.get("driver"); - if (driver != null && !Set.of("postgres", "sqlite").contains(driver)) - throw new Usage("--driver must be postgres or sqlite"); + if (driver != null && !Set.of("mysql", "postgres", "sqlite").contains(driver)) + throw new Usage("--driver must be mysql, postgres, or sqlite"); if (offline && driver != null && !options.values.containsKey("database-url")) url = null; if (url == null || url.isBlank()) { if (!offline) throw new Usage("Set DATABASE_URL or pass --database-url"); - url = "sqlite".equals(driver) ? "jdbc:sqlite::memory:" : "jdbc:postgresql://localhost/unused"; + url = + "mysql".equals(driver) + ? "jdbc:mysql://localhost/unused" + : "sqlite".equals(driver) + ? "jdbc:sqlite::memory:" + : "jdbc:postgresql://localhost/unused"; } - if (!url.startsWith("postgres://") + if (!url.startsWith("mysql://") + && !url.startsWith("jdbc:mysql:") + && !url.startsWith("postgres://") && !url.startsWith("postgresql://") && !url.startsWith("jdbc:postgresql:") && !url.startsWith("sqlite:") && !url.startsWith("jdbc:sqlite:")) - throw new Usage("Expected a Postgres URL or a SQLite JDBC URL"); + throw new Usage("Expected a MySQL, Postgres, or SQLite URL"); Database database; try { database = @@ -55,7 +62,7 @@ private static Database database( throw new Usage("Invalid database URL or schema"); } if (driver != null - && !driver.equals(database.dialect() == Database.Dialect.SQLITE ? "sqlite" : "postgres")) + && !driver.equals(database.dialect().name().toLowerCase(java.util.Locale.ROOT))) throw new Usage("--driver does not match the database URL"); return database; } @@ -101,10 +108,11 @@ private static void help(PrintWriter out, String name, List lines) { out.println(" version Print CLI version"); out.println(); out.println("Connection options:"); - out.println(" --database-url URL Postgres or SQLite JDBC URL; defaults to DATABASE_URL"); + out.println(" --database-url URL MySQL, Postgres, or SQLite URL; defaults to DATABASE_URL"); out.println( - " --driver DRIVER postgres or sqlite; selects dialect for offline SQL export"); - out.println(" --schema NAME Postgres schema (default: connection's current schema)"); + " --driver DRIVER mysql, postgres, or sqlite; selects dialect for offline SQL export"); + out.println( + " --schema NAME MySQL database or Postgres schema (default: connection's current schema)"); out.printf( " --line NAME Migration line: %s (default: main)%n", String.join(", ", lines)); out.println(); diff --git a/java/conformance/pom.xml b/java/conformance/pom.xml index c03b1ac56..bd7c32fb1 100644 --- a/java/conformance/pom.xml +++ b/java/conformance/pom.xml @@ -7,6 +7,7 @@ com.zaxxerHikariCP7.1.0 com.riverqueueriver${project.version} + com.mysqlmysql-connector-j org.postgresqlpostgresql org.xerialsqlite-jdbc diff --git a/java/docs/development.md b/java/docs/development.md index 3c55228f2..c581e60f3 100644 --- a/java/docs/development.md +++ b/java/docs/development.md @@ -37,8 +37,9 @@ fresh schema for each test that is removed afterward. Otherwise they use SQLite. and excludes Postgres-only tests, even if a URL is set in your environment. `test/java/conformance` needs no external database: it runs the JUnit tests tagged `conformance`, using temporary SQLite databases for storage checks even if a -Postgres URL is set. The full SQLite and Postgres targets run these checks -against their selected backend. These targets first generate fresh fixtures +database URL is set. `test/java/conformance/mysql` runs the same tagged tests on +MySQL using `RIVER_TEST_DATABASE_URL`. The full SQLite, Postgres, and MySQL targets +run these checks against their selected backend. These targets first generate fresh fixtures directly from this checkout's Go implementation, just like `test/js/conformance` and `test/rust/conformance`. @@ -80,7 +81,7 @@ matrix running client, worker, and migration tests on JDK 25. All jobs use the shared Java setup action and Maven dependency caching. The whole workflow is filtered to changes in Java, its build configuration, fixtures, and canonical migrations. Go changes run the smaller fixture suite in the shared Conformance -workflow alongside Rust and JavaScript. +workflow on SQLite and MySQL alongside Ruby, Rust, and JavaScript. ## Legacy cross-process harness @@ -119,3 +120,10 @@ does not overwrite the current migrations or generated fixtures. See [validation results](../VALIDATION.md) for the tested revisions and remaining limitations. + +## MySQL tests + +Set `RIVER_TEST_DATABASE_URL=mysql://root@localhost/` and run +`make test/java/mysql` from the repository root. Each test creates and drops its +own database, so the test user needs permission to do both. CI exercises MySQL +8.0 and 8.4 with the shared client, worker, and migration suites. diff --git a/java/pom.xml b/java/pom.xml index b858716ae..1bb7a707f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -13,6 +13,7 @@ tools.jackson.corejackson-databind3.2.3 + com.mysqlmysql-connector-j26.7.0 org.postgresqlpostgresql42.7.12 org.xerialsqlite-jdbc3.53.4.0 org.junit.jupiterjunit-jupiter5.13.4 diff --git a/java/river/pom.xml b/java/river/pom.xml index 5d7582502..e4f7a8989 100644 --- a/java/river/pom.xml +++ b/java/river/pom.xml @@ -6,6 +6,7 @@ Mozilla Public License 2.0https://mozilla.org/MPL/2.0/repo tools.jackson.corejackson-databind + com.mysqlmysql-connector-jtrue org.postgresqlpostgresqltrue org.xerialsqlite-jdbctrue org.junit.jupiterjunit-jupitertest diff --git a/java/river/src/main/java/com/riverqueue/Client.java b/java/river/src/main/java/com/riverqueue/Client.java index a77fb5c7b..7bacb307f 100644 --- a/java/river/src/main/java/com/riverqueue/Client.java +++ b/java/river/src/main/java/com/riverqueue/Client.java @@ -182,9 +182,10 @@ public Job cancel(Connection connection, long id) { var before = locked(c, id); if (before.state().isFinalized()) return before; try (var statement = - Sql.prepare( + Sql.mutation( + database, c, - Sql.query(database, "cancel"), + "cancel", database.timestamp(clock.instant()), Json.encode(clock.instant()), id); @@ -209,7 +210,7 @@ public Job delete(Connection connection, long id) { var job = locked(c, id); if (job.state() == Job.State.RUNNING) throw new RiverException(RiverException.Code.REJECTED, "Cannot delete a running job"); - try (var statement = Sql.prepare(c, Sql.query(database, "delete"), id); + try (var statement = Sql.mutation(database, c, "delete", id); var rows = statement.executeQuery()) { rows.next(); var deleted = read(rows); @@ -256,9 +257,10 @@ public Job complete(Connection connection, Job job, Object metadata) { connection, c -> { try (var statement = - Sql.prepare( + Sql.mutation( + database, c, - Sql.query(database, "complete"), + "complete", "completed", database.timestamp(clock.instant()), "completed", @@ -309,6 +311,13 @@ public List> deleteMany(Connection connection, JobQuery query, boo try (var statement = Sql.prepare(c, sql, params.toArray())) { for (var job : jobs) { statement.setLong(params.size() + 1, job.id()); + if (database.dialect() == Database.Dialect.MYSQL) { + if (statement.executeUpdate() != 0) { + plugin.afterDelete(c, driver(), job); + result.add(job); + } + continue; + } try (var rows = statement.executeQuery()) { // Recheck every filter atomically: a peer may have updated this job since listing. if (!rows.next()) continue; @@ -526,7 +535,9 @@ private Job.InsertResult insertOne( : original.uniqueStates().stream().mapToInt(Job.State::bit).reduce(0, (a, b) -> a | b); boolean postgres = database.dialect() == Database.Dialect.POSTGRES; try { - if (!postgres && key != null && (stateMask & Job.State.of(state).bit()) != 0) { + if (database.dialect() == Database.Dialect.SQLITE + && key != null + && (stateMask & Job.State.of(state).bit()) != 0) { try (var statement = Sql.prepare(connection, Sql.query(database, "writer_lock"))) { statement.executeUpdate(); } @@ -568,13 +579,7 @@ private Job.InsertResult insertOne( key == null ? null : HexFormat.of().parseHex(key), mask }; - String insertSql = Sql.query(database, "insert"); - if (postgres) - insertSql = - insertSql.replace( - "{duplicate}", - Sql.query(database, nonce ? "insert_duplicate_nonce" : "insert_duplicate_xmax")); - try (var statement = Sql.prepare(connection, insertSql, params); + try (var statement = Sql.mutation(database, connection, "insert", params); var rows = statement.executeQuery()) { if (!rows.next()) throw new RiverException(RiverException.Code.REJECTED, "Insertion returned no row"); @@ -585,6 +590,15 @@ private Job.InsertResult insertOne( .path(Protocol.METADATA_UNIQUE_NONCE) .equals(metadata.path(Protocol.METADATA_UNIQUE_NONCE)) : rows.getBoolean("duplicate"); + if (duplicate + && database.dialect() == Database.Dialect.MYSQL + && (key == null + || (stateMask & Job.State.of(state).bit()) == 0 + || !key.equals(job.uniqueKey()) + || job.uniqueStates() == null + || !job.uniqueStates().contains(job.state()))) + throw new RiverException( + RiverException.Code.REJECTED, "Insertion violated an unrelated unique constraint"); if (!duplicate) plugin.afterInsert(connection, driver(), job); if (!duplicate && job.state() == Job.State.AVAILABLE) notifyInsert(connection, job.queue()); return new Job.InsertResult(job, duplicate); @@ -680,8 +694,7 @@ public Job output(Connection connection, long id, Object output) { return atomic( connection, c -> { - try (var statement = - Sql.prepare(c, Sql.query(database, "output"), Json.encode(output), id); + try (var statement = Sql.mutation(database, c, "output", Json.encode(output), id); var rows = statement.executeQuery()) { if (!rows.next()) throw new RiverException(RiverException.Code.NOT_FOUND, "Job " + id + " not found"); @@ -700,9 +713,10 @@ public Job retry(Connection connection, long id) { c -> { var before = locked(c, id); try (var statement = - Sql.prepare( + Sql.mutation( + database, c, - Sql.query(database, "retry"), + "retry", database.timestamp(clock.instant()), id, database.timestamp(clock.instant())); @@ -730,6 +744,9 @@ private void reset() { /** Runs a callback in a new transaction, committing on success and rolling back on failure. */ public T transaction(Transaction action) { try (var connection = database.connection()) { + int isolation = connection.getTransactionIsolation(); + if (database.dialect() == Database.Dialect.MYSQL) + connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); connection.setAutoCommit(false); var previous = ownedCommit.get(); var commit = new Commit(connection); @@ -751,6 +768,8 @@ public T transaction(Transaction action) { } finally { if (previous == null) ownedCommit.remove(); else ownedCommit.set(previous); + if (database.dialect() == Database.Dialect.MYSQL) + connection.setTransactionIsolation(isolation); } } catch (SQLException e) { throw databaseError("River transaction", e); diff --git a/java/river/src/main/java/com/riverqueue/Database.java b/java/river/src/main/java/com/riverqueue/Database.java index a7fc047bd..430747670 100644 --- a/java/river/src/main/java/com/riverqueue/Database.java +++ b/java/river/src/main/java/com/riverqueue/Database.java @@ -44,6 +44,7 @@ public static Database connect(String url, String applicationName) { String jdbc = url.startsWith("jdbc:") ? url : "jdbc:" + url; return new Database(() -> DriverManager.getConnection(jdbc), Dialect.SQLITE, ""); } + boolean mysql = url.startsWith("mysql:") || url.startsWith("jdbc:mysql:"); String jdbc; var properties = new Properties(); properties.setProperty("ApplicationName", applicationName); @@ -59,7 +60,7 @@ public static Database connect(String url, String applicationName) { properties.setProperty("password", URLDecoder.decode(auth[1], StandardCharsets.UTF_8)); } jdbc = - "jdbc:postgresql://" + (mysql ? "jdbc:mysql://" : "jdbc:postgresql://") + uri.getHost() + (uri.getPort() < 0 ? "" : ":" + uri.getPort()) + uri.getRawPath() @@ -67,7 +68,9 @@ public static Database connect(String url, String applicationName) { } String target = jdbc; return new Database( - () -> DriverManager.getConnection(target, properties), Dialect.POSTGRES, ""); + () -> DriverManager.getConnection(target, properties), + mysql ? Dialect.MYSQL : Dialect.POSTGRES, + ""); } public Connection connection() throws SQLException { @@ -117,11 +120,11 @@ public String schema() { } boolean supportsNotifications(Connection connection) throws SQLException { - return dialect == Dialect.SQLITE || capabilities(connection).notifications(); + return dialect != Dialect.POSTGRES || capabilities(connection).notifications(); } boolean usesUniqueNonce(Connection connection) throws SQLException { - return dialect == Dialect.SQLITE || capabilities(connection).uniqueNonce(); + return dialect != Dialect.POSTGRES || capabilities(connection).uniqueNonce(); } private Capabilities capabilities(Connection connection) throws SQLException { @@ -143,6 +146,7 @@ public Database withSchema(String name) { } String prefix() { + if (dialect == Dialect.MYSQL) return schema.isEmpty() ? "" : "`" + schema + "`."; return schema.isEmpty() ? "" : '"' + schema + "\"."; } @@ -150,6 +154,10 @@ Object timestamp(Instant value) { if (value == null) return null; if (dialect == Dialect.POSTGRES) return OffsetDateTime.ofInstant(value.truncatedTo(ChronoUnit.MICROS), ZoneOffset.UTC); + if (dialect == Dialect.MYSQL) + return DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS") + .withZone(ZoneOffset.UTC) + .format(value); value = value.plusNanos(500000).truncatedTo(ChronoUnit.MILLIS); return DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSS") .withZone(ZoneOffset.UTC) @@ -174,6 +182,7 @@ private interface Connector { } public enum Dialect { + MYSQL, POSTGRES, SQLITE } diff --git a/java/river/src/main/java/com/riverqueue/JobQuery.java b/java/river/src/main/java/com/riverqueue/JobQuery.java index e5115e519..68882d81d 100644 --- a/java/river/src/main/java/com/riverqueue/JobQuery.java +++ b/java/river/src/main/java/com/riverqueue/JobQuery.java @@ -216,7 +216,7 @@ private boolean hasMetadataFilter() { return !metadata.isObject() || !metadata.isEmpty(); } - private static String sqliteMetadata( + private static String jsonMetadata( Database database, List parameters, JsonNode wanted, @@ -225,22 +225,28 @@ private static String sqliteMetadata( int depth) { // Array containment is unordered, but each wanted object must match one complete element. // Correlated predicates retain that boundary instead of comparing independent JSON paths. + // MySQL uses scalar subqueries to avoid semijoin rewrites losing nested correlations. String query; if (wanted.isContainer()) { var children = new ArrayList(); String alias = "metadata_" + depth; if (wanted.isObject()) { for (var entry : wanted.properties()) { - parameters.add(entry.getKey()); + parameters.add( + database.dialect() == Database.Dialect.MYSQL + ? "$." + Json.encode(entry.getKey()) + : entry.getKey()); children.add( Sql.query(database, "filter_metadata_member") .replace( "{predicate}", - sqliteMetadata( + jsonMetadata( database, parameters, entry.getValue(), - alias + ".type", + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(COALESCE(JSON_TYPE(" + alias + ".value), 'NULL'))" + : alias + ".type", alias + ".value", depth + 1))); } @@ -250,11 +256,13 @@ private static String sqliteMetadata( Sql.query(database, "filter_metadata_element") .replace( "{predicate}", - sqliteMetadata( + jsonMetadata( database, parameters, element, - alias + ".type", + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(COALESCE(JSON_TYPE(" + alias + ".value), 'NULL'))" + : alias + ".type", alias + ".value", depth + 1))); } @@ -321,6 +329,17 @@ String sql(Database database, List parameters, boolean forDelete) { : field + " " + sort + (descending ? " NULLS FIRST, " : " NULLS LAST, ")) + "id " + sort; + if (database.dialect() == Database.Dialect.MYSQL && !field.isEmpty()) + orderSQL = + field + + " IS NULL " + + (descending ? "DESC" : "ASC") + + ", " + + field + + " " + + sort + + ", id " + + sort; parameters.add(limit); return Sql.query(database, forDelete ? "list_deletable" : "list") .replace("{where}", where) @@ -337,10 +356,19 @@ private String where(Database database, List parameters) { filter(database, filters, parameters, "tags_all", tagsAll); filter(database, filters, parameters, "tags_any", tagsAny); if (hasMetadataFilter()) { - if (database.dialect() == Database.Dialect.SQLITE) - filters.add( - sqliteMetadata(database, parameters, metadata, "json_type(metadata)", "metadata", 0)); - else { + if (database.dialect() != Database.Dialect.POSTGRES) { + String predicate = + jsonMetadata( + database, + parameters, + metadata, + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(JSON_TYPE(metadata))" + : "json_type(metadata)", + "metadata", + 0); + filters.add(predicate); + } else { filters.add(Sql.query(database, "filter_metadata")); parameters.add(Json.encode(metadata)); } diff --git a/java/river/src/main/java/com/riverqueue/Migrator.java b/java/river/src/main/java/com/riverqueue/Migrator.java index 07ac2caaf..6a8190919 100644 --- a/java/river/src/main/java/com/riverqueue/Migrator.java +++ b/java/river/src/main/java/com/riverqueue/Migrator.java @@ -36,14 +36,10 @@ public Migrator(Database database, String line, List migrations) { private void apply(Connection connection, Migration migration, boolean down) throws SQLException { int version = migration.version(); boolean main = line.equals("main"); - if (down && (!main || version > 1)) { - String query = main && version < 5 ? "migration_delete_legacy" : "migration_line_delete"; - Object[] params = main && version < 5 ? new Object[] {version} : new Object[] {line, version}; - try (var statement = Sql.prepare(connection, Sql.query(database, query), params)) { - statement.executeUpdate(); - } - } + boolean mysql = database.dialect() == Database.Dialect.MYSQL; + if (down && !mysql) removeVersion(connection, version); execute(connection, sql(version, down ? Direction.DOWN : Direction.UP)); + if (down && mysql) removeVersion(connection, version); if (!down) { String query = main && version < 5 ? "migration_insert_legacy" : "migration_line_insert"; Object[] params = main && version < 5 ? new Object[] {version} : new Object[] {line, version}; @@ -53,6 +49,18 @@ private void apply(Connection connection, Migration migration, boolean down) thr } } + private void removeVersion(Connection connection, int version) throws SQLException { + boolean main = line.equals("main"); + if (main && version == 1) return; + boolean legacy = + main && (version < 5 || database.dialect() == Database.Dialect.MYSQL && version == 5); + String query = legacy ? "migration_delete_legacy" : "migration_line_delete"; + Object[] params = legacy ? new Object[] {version} : new Object[] {line, version}; + try (var statement = Sql.prepare(connection, Sql.query(database, query), params)) { + statement.executeUpdate(); + } + } + private boolean booleanQuery(Connection connection, String name) throws SQLException { try (var statement = Sql.prepare(connection, Sql.query(database, name)); var rows = statement.executeQuery()) { @@ -65,7 +73,9 @@ private void execute(Connection connection, String sql) throws SQLException { try (var statement = connection.createStatement()) { // Xerial's executeUpdate executes the entire script, including trigger bodies. if (database.dialect() == Database.Dialect.SQLITE) statement.executeUpdate(sql); - else statement.execute(sql); + else if (database.dialect() == Database.Dialect.MYSQL) { + for (String part : MySql.statements(sql)) statement.execute(part); + } else statement.execute(sql); } } @@ -109,8 +119,7 @@ public List list() { } private static List mainMigrations(Database.Dialect dialect) { - String directory = - "migration/" + (dialect == Database.Dialect.POSTGRES ? "postgres/" : "sqlite/"); + String directory = "migration/" + dialect.name().toLowerCase(java.util.Locale.ROOT) + "/"; var migrations = new ArrayList(); for (int i = 0; i < NAMES.size(); i++) { String name = NAMES.get(i); @@ -168,6 +177,10 @@ public Result migrate(Direction direction, Options options) { if (database.dialect() == Database.Dialect.POSTGRES) { execute(connection, Sql.query(database, "migration_lock")); locked = true; + } else if (database.dialect() == Database.Dialect.MYSQL) { + if (!booleanQuery(connection, "migration_lock")) + throw new SQLException("Another migrator holds the database lock"); + locked = true; } var existing = existing(connection); var planned = plan(existing, down, target, maxSteps); diff --git a/java/river/src/main/java/com/riverqueue/MySql.java b/java/river/src/main/java/com/riverqueue/MySql.java new file mode 100644 index 000000000..be4bb6361 --- /dev/null +++ b/java/river/src/main/java/com/riverqueue/MySql.java @@ -0,0 +1,144 @@ +package com.riverqueue; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; +import javax.sql.rowset.RowSetProvider; +import tools.jackson.databind.node.ObjectNode; + +/** MySQL's locking writes and readback; all SQL remains in the query catalog. */ +final class MySql { + private MySql() {} + + static int clean(Database database, Connection connection, Object... horizons) + throws SQLException { + var ids = new ArrayList(); + try (var selected = rows(database, connection, "clean_jobs_select", horizons)) { + while (selected.next()) ids.add(selected.getLong("id")); + } + if (ids.isEmpty()) return 0; + // Lock candidates first, then use primary-key lookups for the delete. A joined + // delete needs a default database, while a subquery delete can lock skipped rows. + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + try (var statement = + Sql.prepare( + connection, + Sql.query(database, "clean_jobs").replace("{ids}", placeholders), + ids.toArray())) { + return statement.executeUpdate(); + } + } + + static ResultSet mutate(Database database, Connection connection, String name, Object[] params) + throws SQLException { + if (connection.getAutoCommit()) + throw new SQLException("MySQL row-returning writes require a transaction"); + return switch (name) { + case "claim", "claim_known" -> claim(database, connection, name, params); + case "delete" -> delete(database, connection, params); + case "insert" -> { + update(database, connection, name, params); + yield rows(database, connection, "inserted"); + } + case "leader_elect" -> { + update(database, connection, name, params); + yield rows(database, connection, "leader_elected", params[0], params[1]); + } + case "complete", "rescue_update" -> { + int idIndex = name.equals("complete") ? 9 : 7; + int metadataIndex = name.equals("complete") ? 6 : 4; + Object[] merged = params.clone(); + try (var before = rows(database, connection, "lock_get", params[idIndex])) { + if (before.next()) { + ObjectNode metadata = (ObjectNode) Json.parse(before.getString("metadata")); + metadata.setAll((ObjectNode) Json.parse((String) params[metadataIndex])); + merged[metadataIndex] = Json.encode(metadata); + } + } + boolean changed = update(database, connection, name, merged) != 0; + yield rows(database, connection, changed ? "lock_get" : "empty_jobs", params[idIndex]); + } + case "cancel", "output", "retry", "queue_update", "schedule_available" -> { + Object id = params[name.equals("retry") ? 1 : params.length - 1]; + boolean changed = update(database, connection, name, params) != 0; + String read = + switch (name) { + case "queue_update" -> changed ? "queue_get" : "empty_queues"; + case "schedule_available" -> changed ? "scheduled_queue" : "empty_queues"; + default -> changed ? "lock_get" : "empty_jobs"; + }; + yield rows(database, connection, read, id); + } + default -> throw new IllegalArgumentException("Unknown MySQL mutation: " + name); + }; + } + + private static ResultSet claim( + Database database, Connection connection, String name, Object[] params) throws SQLException { + var ids = new ArrayList(); + // Keep an indexable queue equality alongside the exact binary comparison so + // concurrent fetches do not briefly lock and skip jobs in unrelated queues. + var selection = new ArrayList<>(Arrays.asList(Arrays.copyOfRange(params, 2, params.length))); + selection.addFirst(params[2]); + try (var selected = rows(database, connection, name + "_select", selection.toArray())) { + while (selected.next()) ids.add(selected.getLong("id")); + } + if (ids.isEmpty()) return rows(database, connection, "empty_jobs", 0); + for (long id : ids) update(database, connection, "claim_one", params[0], params[1], id); + return rows(database, connection, "claimed", Json.encode(ids)); + } + + private static ResultSet delete(Database database, Connection connection, Object[] params) + throws SQLException { + var snapshot = RowSetProvider.newFactory().createCachedRowSet(); + try (var selected = rows(database, connection, "delete_rows", params)) { + snapshot.populate(selected); + } + try { + update(database, connection, "delete", params); + return snapshot; + } catch (SQLException error) { + snapshot.close(); + throw error; + } + } + + private static ResultSet rows( + Database database, Connection connection, String name, Object... params) throws SQLException { + return Sql.queryRows(connection, Sql.query(database, name), params); + } + + private static int update(Database database, Connection connection, String name, Object... params) + throws SQLException { + try (var statement = Sql.prepare(connection, Sql.query(database, name), params)) { + return statement.executeUpdate(); + } + } + + /** Splits migration scripts without enabling multi-statement application queries. */ + static List statements(String script) { + var tokens = + Pattern.compile( + "'(?:''|\\\\.|[^'\\\\])*'|\"(?:\"\"|\\\\.|[^\"\\\\])*\"|`(?:``|[^`])*`|--[^\\n]*|#[^\\n]*|/\\*.*?\\*/|;|.", + Pattern.DOTALL) + .matcher(script); + var statements = new ArrayList(); + var buffer = new StringBuilder(); + while (tokens.find()) { + String token = tokens.group(); + if (token.startsWith("--") || token.startsWith("#") || token.startsWith("/*")) { + buffer.append(' '); + } else if (token.equals(";")) { + if (!buffer.toString().isBlank()) statements.add(buffer.toString().strip()); + buffer.setLength(0); + } else buffer.append(token); + } + if (!buffer.toString().isBlank()) statements.add(buffer.toString().strip()); + return List.copyOf(statements); + } +} diff --git a/java/river/src/main/java/com/riverqueue/Queues.java b/java/river/src/main/java/com/riverqueue/Queues.java index d8fbc4dc7..5da578379 100644 --- a/java/river/src/main/java/com/riverqueue/Queues.java +++ b/java/river/src/main/java/com/riverqueue/Queues.java @@ -114,9 +114,10 @@ private Queue updateWithinTransaction(Connection connection, String name, Object var value = Json.tree(metadata); if (!value.isObject()) throw new IllegalArgumentException("Queue metadata must be an object"); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), connection, - Sql.query(river.database(), "queue_update"), + "queue_update", Json.encode(metadata), river.database().timestamp(Instant.now()), name); diff --git a/java/river/src/main/java/com/riverqueue/Sql.java b/java/river/src/main/java/com/riverqueue/Sql.java index bac27d866..87b0c6a85 100644 --- a/java/river/src/main/java/com/riverqueue/Sql.java +++ b/java/river/src/main/java/com/riverqueue/Sql.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; @@ -12,6 +13,8 @@ final class Sql { private static final Map> QUERIES = Map.of( + Database.Dialect.MYSQL, + load("mysql.sql"), Database.Dialect.POSTGRES, load("postgres.sql"), Database.Dialect.SQLITE, @@ -19,6 +22,63 @@ final class Sql { private Sql() {} + /** Executes a row-returning write within the caller's transaction. */ + static Mutation mutation(Database database, Connection connection, String name, Object... params) + throws SQLException { + return new Mutation(database, connection, name, params); + } + + static final class Mutation implements AutoCloseable { + private final Database database; + private final Connection connection; + private final String name; + private final Object[] params; + private ResultSet rows; + + private Mutation(Database database, Connection connection, String name, Object[] params) { + this.database = database; + this.connection = connection; + this.name = name; + this.params = params; + } + + ResultSet executeQuery() throws SQLException { + if (database.dialect() == Database.Dialect.MYSQL) { + rows = MySql.mutate(database, connection, name, params); + } else { + String sql = query(database, name); + if (name.equals("insert") && database.dialect() == Database.Dialect.POSTGRES) + sql = + sql.replace( + "{duplicate}", + query( + database, + database.usesUniqueNonce(connection) + ? "insert_duplicate_nonce" + : "insert_duplicate_xmax")); + rows = queryRows(connection, sql, params); + } + return rows; + } + + @Override + public void close() throws SQLException { + if (rows != null) rows.close(); + } + } + + static ResultSet queryRows(Connection connection, String sql, Object... params) + throws SQLException { + var statement = prepare(connection, sql, params); + try { + statement.closeOnCompletion(); + return statement.executeQuery(); + } catch (SQLException error) { + statement.close(); + throw error; + } + } + static PreparedStatement prepare(Connection connection, String sql, Object... params) throws SQLException { var statement = connection.prepareStatement(sql); diff --git a/java/river/src/main/java/com/riverqueue/Workers.java b/java/river/src/main/java/com/riverqueue/Workers.java index 4da311738..7b64faeb2 100644 --- a/java/river/src/main/java/com/riverqueue/Workers.java +++ b/java/river/src/main/java/com/riverqueue/Workers.java @@ -349,11 +349,10 @@ private void claim(String queue, int count) { ignored -> { var result = new ArrayList(); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query( - river.database(), - config.fetchOnlyKnownKinds ? "claim_known" : "claim"), + config.fetchOnlyKnownKinds ? "claim_known" : "claim", config.fetchOnlyKnownKinds ? new Object[] { river.database().timestamp(Instant.now()), @@ -663,8 +662,7 @@ private void completeBatches() { var rows = new ArrayList>(); for (var completion : batch) try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "complete"), completion.params); + Sql.mutation(river.database(), c, "complete", completion.params); var result = statement.executeQuery()) { if (result.next()) rows.add(river.readPartial(result).job()); else { @@ -722,8 +720,8 @@ private void heartbeat(String queue) { } private void listen() { - if (river.database().dialect() == Database.Dialect.SQLITE) { - listenSqlite(); + if (river.database().dialect() != Database.Dialect.POSTGRES) { + listenOutbox(); return; } while (running.get()) { @@ -785,7 +783,7 @@ private void deliver(String topic, String payload) { } } - private void listenSqlite() { + private void listenOutbox() { long after = 0; boolean initialized = false; while (running.get()) { @@ -797,8 +795,9 @@ private void listenSqlite() { try (var statement = Sql.prepare(c, Sql.query(river.database(), "notification_cursor")); var rows = statement.executeQuery()) { - rows.next(); - return rows.getLong(1); + long cursor = 0; + while (rows.next()) cursor = rows.getLong(1); + return cursor; } }); initialized = true; @@ -929,9 +928,10 @@ private void maintain() { } } try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "leader_elect"), + "leader_elect", id, river.database().timestamp(now), river.database().timestamp(expiresAt)); @@ -1076,11 +1076,18 @@ private void clean(Instant now) { config.discardedRetention.isNegative() ? null : river.database().timestamp(now.minus(config.discardedRetention)); - try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "clean_jobs"), cancelled, completed, discarded)) { - statement.executeUpdate(); - } + if (river.database().dialect() == Database.Dialect.MYSQL) + MySql.clean(river.database(), c, cancelled, completed, discarded); + else + try (var statement = + Sql.prepare( + c, + Sql.query(river.database(), "clean_jobs"), + cancelled, + completed, + discarded)) { + statement.executeUpdate(); + } try (var statement = Sql.prepare( c, @@ -1093,7 +1100,7 @@ private void clean(Instant now) { } private void cleanNotifications(Instant now) { - if (river.database().dialect() != Database.Dialect.SQLITE) return; + if (river.database().dialect() == Database.Dialect.POSTGRES) return; river.transaction( c -> { try (var statement = @@ -1158,13 +1165,12 @@ private void schedule(Instant now) { try { if (!collision) try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "schedule_available"), job.id()); + Sql.mutation(river.database(), c, "schedule_available", job.id()); var rows = statement.executeQuery()) { if (rows.next()) river.notifyInsert(c, rows.getString(1)); } } catch (SQLException error) { - if (!"23505".equals(error.getSQLState())) throw error; + if (!"23505".equals(error.getSQLState()) && error.getErrorCode() != 1062) throw error; c.rollback(savepoint); collision = true; } finally { @@ -1256,9 +1262,10 @@ private void rescue(Instant now, Instant term) { Protocol.METADATA_RESCUE_COUNT, job.metadata().path(Protocol.METADATA_RESCUE_COUNT).asLong(0) + 1)); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "rescue_update"), + "rescue_update", state.value(), river.database().timestamp(state.isFinalized() ? now : null), river.database().timestamp(scheduled), diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql new file mode 100644 index 000000000..9aee5ff2f --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql new file mode 100644 index 000000000..c680cb3db --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify is +-- omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql new file mode 100644 index 000000000..7725c9f12 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. MySQL doesn't have enum types to +-- alter, so we update the CHECK constraint instead. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql new file mode 100644 index 000000000..108c807a8 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql new file mode 100644 index 000000000..fc5889c45 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..c4c740ea7 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..d5dc9a6b4 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/java/river/src/main/resources/com/riverqueue/mysql.sql b/java/river/src/main/resources/com/riverqueue/mysql.sql new file mode 100644 index 000000000..8e9903c76 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/mysql.sql @@ -0,0 +1,294 @@ +-- name: columns +id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states + +-- name: get +SELECT {columns} FROM {schema}river_job WHERE id = ? + +-- name: insert +INSERT INTO {schema}river_job +(args, created_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) +VALUES (CAST(? AS JSON), ?, ?, ?, CAST(? AS JSON), ?, ?, ?, ?, CAST(? AS JSON), ?, ?) +ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) + +-- name: notify +INSERT INTO {schema}river_notification (topic, payload) VALUES (?, ?) + +-- name: migration_insert +INSERT INTO {schema}river_migration (line,version) VALUES ('main',?) + +-- name: migration_insert_legacy +INSERT INTO {schema}river_migration (version) VALUES (?) + +-- name: migration_delete +DELETE FROM {schema}river_migration WHERE line='main' AND version=? + +-- name: migration_delete_legacy +DELETE FROM {schema}river_migration WHERE version=? + +-- name: migration_versions +SELECT version FROM {schema}river_migration WHERE line='main' ORDER BY version + +-- name: migration_versions_legacy +SELECT version FROM {schema}river_migration ORDER BY version + +-- name: list +SELECT {columns} FROM {schema}river_job WHERE {where} ORDER BY {order} LIMIT ? + +-- name: list_deletable +SELECT {columns} FROM {schema}river_job WHERE {where} AND state != 'running' ORDER BY {order} LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: cursor_id +(id {comparison} ?) + +-- name: cursor_null_asc +({field} IS NULL AND id > ?) + +-- name: cursor_null_desc +({field} IS NOT NULL OR id < ?) + +-- name: cursor_time +({field} {comparison} ? OR ({field} = ? AND id {comparison} ?)) + +-- name: cursor_time_nullable +({field} > ? OR ({field} = ? AND id > ?) OR {field} IS NULL) + +-- name: delete +DELETE FROM {schema}river_job WHERE id=? AND state != 'running' + +-- name: delete_filtered +DELETE FROM {schema}river_job WHERE {where} AND id=? AND state != 'running' + +-- name: retry +UPDATE {schema}river_job SET finalized_at=NULL, +max_attempts=CASE WHEN attempt=max_attempts THEN max_attempts+1 ELSE max_attempts END, scheduled_at=?, state='available' +WHERE id=? AND state != 'running' AND NOT (state='available' AND scheduled_at < ?) + +-- name: reset_jobs +DELETE FROM {schema}river_job + +-- name: reset_queues +DELETE FROM {schema}river_queue + +-- name: reset_leader +DELETE FROM {schema}river_leader + +-- name: reset_notifications +DELETE FROM {schema}river_notification + +-- name: migration_exists +SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=COALESCE(NULLIF('{schema_name}',''),DATABASE()) AND table_name='river_migration') + +-- name: migration_has_line +SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=COALESCE(NULLIF('{schema_name}',''),DATABASE()) AND table_name='river_migration' AND column_name='line') + +-- name: lock_get +SELECT {columns} FROM {schema}river_job WHERE id=? FOR UPDATE + +-- name: cancel +UPDATE {schema}river_job SET +finalized_at=CASE WHEN state='running' THEN finalized_at ELSE ? END, +metadata=JSON_SET(metadata, '$.cancel_attempted_at', CAST(? AS JSON)), +state=CASE WHEN state='running' THEN state ELSE 'cancelled' END +WHERE id=? AND state NOT IN ('cancelled','completed','discarded') AND finalized_at IS NULL + +-- name: output +UPDATE {schema}river_job SET metadata=JSON_SET(metadata,'$.output',CAST(? AS JSON)) WHERE id=? + +-- name: filter_tags_all +JSON_CONTAINS(tags, CAST(? AS JSON)) + +-- name: filter_tags_any +JSON_OVERLAPS(tags, CAST(? AS JSON)) + +-- name: filter_ids +id IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value BIGINT PATH '$')) AS wanted) + +-- name: filter_kinds +CAST(kind AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PATH '$')) AS wanted) + +-- name: filter_priorities +priority IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value SMALLINT PATH '$')) AS wanted) + +-- name: filter_queues +CAST(queue AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PATH '$')) AS wanted) + +-- name: filter_states +state IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PATH '$')) AS wanted) + +-- name: queue_pause +UPDATE {schema}river_queue SET updated_at=CASE WHEN paused_at IS NULL THEN ? ELSE updated_at END, paused_at=COALESCE(paused_at, UTC_TIMESTAMP(6)) WHERE (?='*' OR name=CAST(? AS BINARY)) + +-- name: queue_resume +UPDATE {schema}river_queue SET updated_at=CASE WHEN paused_at IS NOT NULL THEN ? ELSE updated_at END, +paused_at=NULL WHERE (?='*' OR name=CAST(? AS BINARY)) + +-- name: queue_heartbeat +INSERT INTO {schema}river_queue (name,created_at,updated_at) VALUES (?,?,?) ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at) + +-- name: leader_get +SELECT leader_id,elected_at FROM {schema}river_leader WHERE name='default' + +-- name: leader_elect +INSERT INTO {schema}river_leader (name,leader_id,elected_at,expires_at) VALUES ('default',?,?,?) ON DUPLICATE KEY UPDATE name=name + +-- name: leader_expire +DELETE FROM {schema}river_leader WHERE expires_at < ? + +-- name: leader_renew +UPDATE {schema}river_leader SET expires_at=? WHERE leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at >= ? + +-- name: leader_resign +DELETE FROM {schema}river_leader WHERE leader_id=CAST(? AS BINARY) AND elected_at=? + +-- name: notifications +SELECT id,topic,payload FROM {schema}river_notification WHERE id>? ORDER BY id LIMIT 256 FOR SHARE + +-- name: queue_get +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue WHERE name=? + +-- name: queue_list +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue ORDER BY name LIMIT ? + +-- name: queue_update +UPDATE {schema}river_queue SET metadata=CAST(? AS JSON), updated_at=? WHERE name=? + +-- name: complete +WITH input AS (SELECT ? AS target_state, ? AS now, ? AS state, ? AS finalized_at, +? AS scheduled_at, ? AS attempt, CAST(? AS JSON) AS metadata, ? AS error_present, CAST(? AS JSON) AS error, +? AS id, ? AS old_attempt, ? AS attempted_at) +UPDATE {schema}river_job j JOIN input i ON j.id=i.id SET +j.errors=CASE WHEN j.state != 'running' OR i.error_present IS NULL THEN j.errors ELSE JSON_ARRAY_APPEND(CASE WHEN j.errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(j.errors)='ARRAY' THEN j.errors ELSE JSON_ARRAY(j.errors) END,'$',i.error) END, +j.finalized_at=CASE WHEN j.state != 'running' THEN j.finalized_at WHEN i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at') THEN i.now ELSE i.finalized_at END, +j.scheduled_at=CASE WHEN j.state != 'running' OR (i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at')) THEN j.scheduled_at ELSE i.scheduled_at END, +j.attempt=CASE WHEN j.state != 'running' OR (i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at')) THEN j.attempt ELSE i.attempt END, +j.state=CASE WHEN j.state != 'running' THEN j.state WHEN i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at') THEN 'cancelled' ELSE i.state END, +j.metadata=i.metadata +WHERE j.attempt=i.old_attempt AND j.attempted_at=i.attempted_at + +-- name: delete_finalized +DELETE FROM {schema}river_job WHERE state IN ('cancelled','completed','discarded') AND finalized_at? ORDER BY id LIMIT 1000 FOR UPDATE SKIP LOCKED + +-- name: cancel_requested +SELECT id FROM {schema}river_job WHERE state='running' AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') + +-- name: notification_cursor +SELECT id FROM {schema}river_notification ORDER BY id FOR UPDATE + +-- name: schedule_select +SELECT id,unique_key FROM {schema}river_job WHERE state IN ('scheduled','retryable') AND scheduled_at<=? ORDER BY priority,scheduled_at,id LIMIT 10000 FOR UPDATE SKIP LOCKED + +-- name: schedule_collision +SELECT id FROM {schema}river_job WHERE id<>? AND unique_key=? AND unique_states IS NOT NULL AND (unique_states & CASE state WHEN 'available' THEN 1 WHEN 'cancelled' THEN 2 WHEN 'completed' THEN 4 WHEN 'discarded' THEN 8 WHEN 'pending' THEN 16 WHEN 'retryable' THEN 32 WHEN 'running' THEN 64 WHEN 'scheduled' THEN 128 ELSE 0 END) != 0 FOR UPDATE + +-- name: schedule_available +UPDATE {schema}river_job SET state='available' WHERE id=? AND state IN ('scheduled','retryable') + +-- name: schedule_discard +UPDATE {schema}river_job SET finalized_at=?, +metadata=JSON_SET(metadata, '$.unique_key_conflict', 'scheduler_discarded'), state='discarded' +WHERE id=? AND state IN ('scheduled','retryable') + +-- name: clean_jobs +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM {schema}river_job WHERE id IN ({ids}) + +-- name: clean_jobs_select +SELECT id FROM {schema}river_job +WHERE (state='cancelled' AND finalized_at'main') + +-- name: inserted +SELECT {columns} FROM {schema}river_job WHERE id = LAST_INSERT_ID() FOR UPDATE + +-- name: delete_rows +SELECT {columns} FROM {schema}river_job WHERE id=? AND state != 'running' FOR UPDATE + +-- name: empty_jobs +SELECT {columns} FROM {schema}river_job WHERE id=? AND false + +-- name: empty_queues +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue WHERE name=? AND false + +-- name: filter_metadata +JSON_CONTAINS(metadata, CAST(? AS JSON)) + +-- name: leader_elected +SELECT elected_at FROM {schema}river_leader WHERE leader_id=CAST(? AS BINARY) AND elected_at=? FOR UPDATE + +-- name: scheduled_queue +SELECT queue FROM {schema}river_job WHERE id=? FOR UPDATE + +-- name: migration_lock +SELECT GET_LOCK(CONCAT('river_migrate:',COALESCE(NULLIF('{schema_name}',''),DATABASE())),0) + +-- name: migration_unlock +SELECT RELEASE_LOCK(CONCAT('river_migrate:',COALESCE(NULLIF('{schema_name}',''),DATABASE()))) + +-- name: create_schema +CREATE DATABASE IF NOT EXISTS `{schema_name}` + +-- name: claim_select +SELECT id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? +AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL) +ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: claim_known_select +SELECT id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? +AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL) +AND CAST(kind AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin PATH '$')) AS wanted) +ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: claim_one +UPDATE {schema}river_job SET attempted_at=?, +attempted_by=JSON_ARRAY_APPEND(CASE WHEN JSON_LENGTH(attempted_by)>=100 THEN JSON_EXTRACT(attempted_by, '$[last-98 to last]') ELSE COALESCE(attempted_by,JSON_ARRAY()) END, '$', ?), +attempt=attempt+1, state='running' WHERE id=? + +-- name: claimed +SELECT {columns} FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value BIGINT PATH '$')) AS wanted STRAIGHT_JOIN {schema}river_job FORCE INDEX (PRIMARY) ON river_job.id=wanted.value ORDER BY priority,scheduled_at,id + +-- name: filter_metadata_container +({type}='{kind}' AND {children}) + +-- name: filter_metadata_element +(SELECT 1 FROM JSON_TABLE(CASE WHEN {type}='{kind}' THEN {value} END, '$[*]' COLUMNS(value JSON PATH '$')) AS {alias} WHERE {predicate} LIMIT 1) IS NOT NULL + +-- name: filter_metadata_member +(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT(CASE WHEN {type}='{kind}' THEN {value} END, ?), '$' COLUMNS(value JSON PATH '$')) AS {alias} WHERE {predicate} LIMIT 1) IS NOT NULL + +-- name: filter_metadata_scalar +(SELECT 1 FROM JSON_TABLE(CAST(? AS JSON), '$' COLUMNS(value JSON PATH '$')) AS wanted +WHERE {type} NOT IN ('object','array') AND ({value} IS NULL AND wanted.value IS NULL OR JSON_CONTAINS({value}, wanted.value)) LIMIT 1) IS NOT NULL diff --git a/java/river/src/test/java/com/riverqueue/ClientTest.java b/java/river/src/test/java/com/riverqueue/ClientTest.java index fadbe9574..740b4f63c 100644 --- a/java/river/src/test/java/com/riverqueue/ClientTest.java +++ b/java/river/src/test/java/com/riverqueue/ClientTest.java @@ -48,6 +48,49 @@ void applicationTransactionOwnsCommitAndRollback() throws Exception { } } + @Test + void cleanupSkipsLockedJobs() throws Exception { + var client = database(); + var database = client.database(); + org.junit.jupiter.api.Assumptions.assumeTrue(database.dialect() == Database.Dialect.MYSQL); + var locked = client.insert(EMAIL, new Email("locked@example.com")).job(); + var unlocked = client.insert(EMAIL, new Email("unlocked@example.com")).job(); + for (int i = 0; i < 999; i++) client.insert(EMAIL, new Email("expired@example.com")); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET state='completed', finalized_at='1970-01-01 00:00:00'"); + } + Client.Transaction clean = + connection -> { + try (var statement = connection.createStatement()) { + statement.execute("SET SESSION innodb_lock_wait_timeout=2"); + } + return MySql.clean( + database, + connection, + database.timestamp(NOW), + database.timestamp(NOW), + database.timestamp(NOW)); + }; + client.transaction( + connection -> { + try (var statement = + Sql.prepare(connection, Sql.query(database, "lock_get"), locked.id()); + var rows = statement.executeQuery()) { + assertTrue(rows.next()); + assertEquals(1000, client.transaction(clean)); + } + assertThrows(RiverException.class, () -> client.get(unlocked.id())); + assertEquals(locked.id(), client.get(locked.id()).id()); + return null; + }); + assertEquals(1, client.transaction(clean)); + assertTrue(client.list(JobQuery.all()).jobs().isEmpty()); + } + @Test void crossKindUniqueConflictsPreserveTheIncumbent() { var client = database(); @@ -140,9 +183,10 @@ void delayedCompletionPreservesARescheduledJob(Job.State state) { client.transaction( connection -> { try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", database.timestamp(NOW), "old-worker", "default", @@ -436,6 +480,10 @@ void typedGetAndMixedBatchRetainArgumentKinds() throws Exception { @Test void checkedHookFailureRollsBackInsideCallerTransaction() throws Exception { var base = database(); + try (var connection = base.database().connection(); + var statement = connection.createStatement()) { + statement.execute("CREATE TABLE " + base.database().prefix() + "hook_write (id INTEGER)"); + } var river = base.withExtension( new Extension() { @@ -445,7 +493,7 @@ public void beforeInsert( throws java.sql.SQLException { try (var statement = connection.createStatement()) { statement.execute( - "CREATE TABLE " + base.database().prefix() + "hook_write (id INTEGER)"); + "INSERT INTO " + base.database().prefix() + "hook_write VALUES (1)"); } throw new java.sql.SQLException("hook rejected"); } @@ -460,15 +508,12 @@ public void beforeInsert( connection, List.of(EMAIL.submission(new Email("rejected@example.com"))))); assertEquals(RiverException.Code.DATABASE, failure.code()); assertInstanceOf(java.sql.SQLException.class, failure.getCause()); - try (var rows = - connection - .getMetaData() - .getTables( - null, - river.database().schema().isEmpty() ? null : river.database().schema(), - "hook_write", - null)) { - assertFalse(rows.next()); + try (var statement = connection.createStatement(); + var rows = + statement.executeQuery( + "SELECT count(*) FROM " + base.database().prefix() + "hook_write")) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1)); } var kept = base.insert(connection, EMAIL, new Email("kept@example.com")).job(); connection.commit(); @@ -706,6 +751,40 @@ void uniqueByFieldsSelectsJsonNamesWithoutSplittingDots() { assertNotEquals(first.job().id(), different.job().id()); } + @ParameterizedTest + @ValueSource(strings = {"inactive_existing", "inactive_incoming"}) + void unrelatedUniqueConflictsAreRejectedWhenReinserting(String mode) throws Exception { + var client = database(); + var database = client.database(); + org.junit.jupiter.api.Assumptions.assumeTrue(database.dialect() == Database.Dialect.MYSQL); + var inserted = + client + .insert( + EMAIL, + new Email("unique@example.com"), + InsertOptions.builder().unique(Unique.args()).build()) + .job(); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "CREATE UNIQUE INDEX custom_kind ON " + database.prefix() + "river_job(kind)"); + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET state='running', unique_states=" + + (mode.equals("inactive_incoming") ? "64" : "NULL")); + } + var original = client.get(inserted.id()); + // Reinsertion uses available; either its key or the existing key is inactive. + var error = + assertThrows( + RiverException.class, + () -> client.transaction(connection -> client.driver().reinsert(connection, original))); + assertTrue(error.getMessage().contains("unrelated unique constraint")); + assertEquals(original, client.get(original.id())); + assertEquals(1, client.list(JobQuery.all()).jobs().size()); + } + record Report(int accountId) {} record Email(String address) {} diff --git a/java/river/src/test/java/com/riverqueue/MySqlTest.java b/java/river/src/test/java/com/riverqueue/MySqlTest.java new file mode 100644 index 000000000..0ca8a122f --- /dev/null +++ b/java/river/src/test/java/com/riverqueue/MySqlTest.java @@ -0,0 +1,78 @@ +package com.riverqueue; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +@EnabledIfSystemProperty(named = "river.test.database", matches = "mysql") +class MySqlTest { + @TempDir Path directory; + @RegisterExtension final TestDatabase databases = new TestDatabase(); + + @Test + void matchesKindsAndQueuesExactly() throws Exception { + var database = databases.open(directory.resolve("unused")); + var client = new Client(database); + var type = JobType.of("message", String.class); + var wanted = client.insert(type, "first").job(); + var padded = client.insert(type, "second").job(); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET kind='message ', queue='default ' WHERE id=" + + padded.id()); + } + assertEquals( + List.of(wanted.id()), + client.list(JobQuery.builder().kinds("message").build()).jobs().stream() + .map(Job::id) + .toList()); + assertEquals( + List.of(wanted.id()), + client.list(JobQuery.builder().queues("default").build()).jobs().stream() + .map(Job::id) + .toList()); + } + + @Test + void migrationRoundTrip() { + var database = databases.open(directory.resolve("unused")); + var migrator = new Migrator(database); + assertTrue(migrator.migrate().valid()); + assertEquals( + List.of(8, 7, 6, 5, 4, 3, 2, 1), + migrator + .migrate(Migrator.Direction.DOWN, Migrator.Options.defaults().targetVersion(0)) + .applied()); + assertTrue(migrator.list().stream().noneMatch(Migrator.Status::applied)); + assertTrue(migrator.migrate().valid()); + } + + @Test + void largeIdsRemainDistinct() throws Exception { + var database = databases.open(directory.resolve("unused")); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "ALTER TABLE " + database.prefix() + "river_job AUTO_INCREMENT=9007199254740993"); + } + var client = new Client(database); + var type = JobType.of("message", String.class); + var first = client.insert(type, "first").job(); + var second = client.insert(type, "second").job(); + assertEquals(9007199254740993L, first.id()); + assertEquals(first.id() + 1, second.id()); + assertEquals( + List.of(second.id()), + client.list(JobQuery.builder().ids(second.id()).build()).jobs().stream() + .map(Job::id) + .toList()); + } +} diff --git a/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java b/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java index 0f5698572..70ae39d90 100644 --- a/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java +++ b/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java @@ -74,9 +74,10 @@ public JsonNode decode(Job job) { var jobs = new ArrayList(); var now = database.timestamp(Instant.now()); try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", now, "batch", "peers", diff --git a/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java b/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java index e7206c185..efdd87690 100644 --- a/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java +++ b/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java @@ -95,7 +95,11 @@ public boolean clean(Client client, Instant now) { "UPDATE " + database.prefix() + "river_job SET state = 'running', attempt = 3, attempted_at = ?, errors = " - + (postgres ? "ARRAY[?::jsonb]" : "json_array(json(?))") + + (switch (database.dialect()) { + case MYSQL -> "JSON_ARRAY(CAST(? AS JSON))"; + case POSTGRES -> "ARRAY[?::jsonb]"; + case SQLITE -> "json_array(json(?))"; + }) + " WHERE id = ?", database.timestamp(Instant.EPOCH), Json.encode(fixture.required("attempt_error")), @@ -119,7 +123,7 @@ public boolean clean(Client client, Instant now) { .errorHandler(errors::add) .retryPolicy( job -> { - if (!postgres) { + if (database.dialect() == Database.Dialect.SQLITE) { // Retry selection is between rescue's read and update. A second writer // must already be excluded, or its commit would invalidate that snapshot. try (var connection = database.connection(); @@ -138,7 +142,7 @@ public boolean clean(Client client, Instant now) { .start(); var subscription = workers.subscribe(completed::complete, Workers.EventKind.JOB_COMPLETED)) { - if (!postgres) { + if (database.dialect() == Database.Dialect.SQLITE) { var conflict = writeConflict.get(5, TimeUnit.SECONDS); assertNotNull(conflict, "Rescue must reserve the writer before selecting jobs"); assertEquals(5, conflict.getErrorCode(), "Expected SQLITE_BUSY for the competing writer"); diff --git a/java/river/src/test/java/com/riverqueue/SchedulingTest.java b/java/river/src/test/java/com/riverqueue/SchedulingTest.java index 85a6be311..e00889dd1 100644 --- a/java/river/src/test/java/com/riverqueue/SchedulingTest.java +++ b/java/river/src/test/java/com/riverqueue/SchedulingTest.java @@ -84,7 +84,9 @@ public void afterInsert( assertNotNull(second); assertEquals(java.util.Set.of("periodic", "scheduled"), java.util.Set.of(first, second)); assertFalse(errors.isEmpty()); - assertTrue(errors.stream().allMatch(error -> "periodic failed".equals(error.getMessage()))); + assertTrue( + errors.stream().allMatch(error -> "periodic failed".equals(error.getMessage())), + () -> "Unexpected errors: " + errors); } } diff --git a/java/river/src/test/java/com/riverqueue/TestDatabase.java b/java/river/src/test/java/com/riverqueue/TestDatabase.java index 365efd588..b1c2a9f1c 100644 --- a/java/river/src/test/java/com/riverqueue/TestDatabase.java +++ b/java/river/src/test/java/com/riverqueue/TestDatabase.java @@ -16,7 +16,10 @@ public void afterEach(ExtensionContext context) throws Exception { for (var database : databases) try (var connection = database.connection(); var statement = connection.createStatement()) { - statement.execute("DROP SCHEMA IF EXISTS \"" + database.schema() + "\" CASCADE"); + statement.execute( + database.dialect() == Database.Dialect.MYSQL + ? "DROP DATABASE IF EXISTS `" + database.schema() + "`" + : "DROP SCHEMA IF EXISTS \"" + database.schema() + "\" CASCADE"); } } @@ -24,14 +27,14 @@ Database open(Path file) { String url = System.getenv("RIVER_TEST_DATABASE_URL"); String backend = System.getProperty("river.test.database", url == null ? "sqlite" : "postgres"); if (backend.equals("sqlite")) return sqlite(file); - if (!backend.equals("postgres")) + if (!backend.equals("postgres") && !backend.equals("mysql")) throw new IllegalArgumentException("Unknown test backend: " + backend); if (url == null || url.isBlank()) throw new IllegalStateException("RIVER_TEST_DATABASE_URL is required for Postgres tests"); var database = Database.connect(url).withSchema("java_" + UUID.randomUUID().toString().replace("-", "")); - if (database.dialect() != Database.Dialect.POSTGRES) - throw new IllegalArgumentException("RIVER_TEST_DATABASE_URL must select Postgres"); + if (!database.dialect().name().equalsIgnoreCase(backend)) + throw new IllegalArgumentException("RIVER_TEST_DATABASE_URL must select " + backend); databases.add(database); new Migrator(database).migrate(); return database; diff --git a/java/river/src/test/java/com/riverqueue/WorkersTest.java b/java/river/src/test/java/com/riverqueue/WorkersTest.java index e4fe2d80b..8500bf631 100644 --- a/java/river/src/test/java/com/riverqueue/WorkersTest.java +++ b/java/river/src/test/java/com/riverqueue/WorkersTest.java @@ -530,9 +530,10 @@ void goResignationWakesLeadershipBeforeNextPoll() throws Exception { connection -> { var now = Instant.now(); try (var statement = - Sql.prepare( + Sql.mutation( + client.database(), connection, - Sql.query(client.database(), "leader_elect"), + "leader_elect", leaderId, client.database().timestamp(now), client.database().timestamp(now.plusSeconds(3600))); diff --git a/job_list_params.go b/job_list_params.go index 4400c1505..ea4187e97 100644 --- a/job_list_params.go +++ b/job_list_params.go @@ -179,6 +179,8 @@ type JobListParams struct { where []dblist.WherePredicate } +const jobListMetadataPredicatePostgres = `metadata @> @metadata_fragment::jsonb` + // NewJobListParams creates a new JobListParams to return available jobs sorted // by time in ascending order, returning 100 jobs at most. func NewJobListParams() *JobListParams { @@ -357,9 +359,9 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { // first descending. case p.after.time.IsZero(): if sortOrder == dblist.SortOrderAsc { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" IS NULL AND "id" > @after_id)`, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s IS NULL AND id > @after_id)`, timeField)}) } else { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" IS NOT NULL OR "id" < @after_id)`, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s IS NOT NULL OR id < @after_id)`, timeField)}) } default: @@ -367,11 +369,11 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { if sortOrder == dblist.SortOrderAsc { var orNull string if timeFieldNullable { - orNull = fmt.Sprintf(` OR "%s" IS NULL`, timeField) + orNull = fmt.Sprintf(` OR %s IS NULL`, timeField) } - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" > @cursor_time OR ("%s" = @cursor_time AND "id" > @after_id)%s)`, timeField, timeField, orNull)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s > @cursor_time OR (%s = @cursor_time AND id > @after_id)%s)`, timeField, timeField, orNull)}) } else { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" < @cursor_time OR ("%s" = @cursor_time AND "id" < @after_id))`, timeField, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s < @cursor_time OR (%s = @cursor_time AND id < @after_id))`, timeField, timeField)}) } } } @@ -391,6 +393,22 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { }, nil } +func (p *JobListParams) withMetadataPredicateSQL(buildFunc func(string, string, string) (string, map[string]any, error)) (*JobListParams, error) { + paramsCopy := p.copy() + for i := range paramsCopy.where { + predicate := ¶msCopy.where[i] + if predicate.SQL == jobListMetadataPredicatePostgres { + fragment, _ := predicate.NamedArgs["metadata_fragment"].(string) + var err error + predicate.SQL, predicate.NamedArgs, err = buildFunc("metadata", "metadata_fragment", fragment) + if err != nil { + return nil, err + } + } + } + return paramsCopy, nil +} + // After returns an updated filter set that will only return jobs // after the given cursor. func (p *JobListParams) After(cursor *JobListCursor) *JobListParams { @@ -452,7 +470,7 @@ func (p *JobListParams) Metadata(json string) *JobListParams { paramsCopy.metadataCalled = true paramsCopy.where = append(paramsCopy.where, dblist.WherePredicate{ NamedArgs: map[string]any{"metadata_fragment": json}, - SQL: `metadata @> @metadata_fragment::jsonb`, + SQL: jobListMetadataPredicatePostgres, }) return paramsCopy } diff --git a/job_list_params_test.go b/job_list_params_test.go index e964a5975..3a2fdce61 100644 --- a/job_list_params_test.go +++ b/job_list_params_test.go @@ -429,7 +429,7 @@ func Test_JobListParams_toDBParams_CustomConditions(t *testing.T) { Where("state = @other_state OR finalized_at IS NULL", NamedArgs{"other_state": "completed"}). After(&JobListCursor{id: 42, time: cursorTime}) driverParams := driverParamsFunc(t, bundle) - const cursorSQL = `("finalized_at" < @cursor_time OR ("finalized_at" = @cursor_time AND "id" < @after_id))` + const cursorSQL = `(finalized_at < @cursor_time OR (finalized_at = @cursor_time AND id < @after_id))` require.Equal(t, "state = any(@state)\n AND state = @other_state OR finalized_at IS NULL\n AND "+cursorSQL, driverParams.WhereClause) require.Equal(t, map[string]any{ "after_id": int64(42), @@ -530,29 +530,29 @@ func Test_JobListParams_toDBParamsNullableTimeField(t *testing.T) { name: "AttemptedAtAsc", params: NewJobListParams().States(rivertype.JobStateRunning).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "attempted_at ASC NULLS LAST, id ASC", - wantAfterNull: `("attempted_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("attempted_at" > @cursor_time OR ("attempted_at" = @cursor_time AND "id" > @after_id) OR "attempted_at" IS NULL)`, + wantAfterNull: `(attempted_at IS NULL AND id > @after_id)`, + wantAfterTime: `(attempted_at > @cursor_time OR (attempted_at = @cursor_time AND id > @after_id) OR attempted_at IS NULL)`, }, { name: "AttemptedAtDesc", params: NewJobListParams().States(rivertype.JobStateRunning).OrderBy(JobListOrderByTime, SortOrderDesc), wantOrderBy: "attempted_at DESC NULLS FIRST, id DESC", - wantAfterNull: `("attempted_at" IS NOT NULL OR "id" < @after_id)`, - wantAfterTime: `("attempted_at" < @cursor_time OR ("attempted_at" = @cursor_time AND "id" < @after_id))`, + wantAfterNull: `(attempted_at IS NOT NULL OR id < @after_id)`, + wantAfterTime: `(attempted_at < @cursor_time OR (attempted_at = @cursor_time AND id < @after_id))`, }, { name: "FinalizedAtMixedStatesAsc", params: NewJobListParams().States(rivertype.JobStateCompleted, rivertype.JobStateAvailable).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterNull: `("finalized_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id) OR "finalized_at" IS NULL)`, + wantAfterNull: `(finalized_at IS NULL AND id > @after_id)`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id) OR finalized_at IS NULL)`, }, { name: "FinalizedAtMixedStatesDesc", params: NewJobListParams().States(rivertype.JobStateCompleted, rivertype.JobStateAvailable).OrderBy(JobListOrderByTime, SortOrderDesc), wantOrderBy: "finalized_at DESC NULLS FIRST, id DESC", - wantAfterNull: `("finalized_at" IS NOT NULL OR "id" < @after_id)`, - wantAfterTime: `("finalized_at" < @cursor_time OR ("finalized_at" = @cursor_time AND "id" < @after_id))`, + wantAfterNull: `(finalized_at IS NOT NULL OR id < @after_id)`, + wantAfterTime: `(finalized_at < @cursor_time OR (finalized_at = @cursor_time AND id < @after_id))`, }, { // An ungrouped OR can include available jobs despite the completed @@ -560,20 +560,20 @@ func Test_JobListParams_toDBParamsNullableTimeField(t *testing.T) { name: "FinalizedAtWithCondition", params: NewJobListParams().States(rivertype.JobStateCompleted).OrderBy(JobListOrderByTime, SortOrderAsc).Where("state = 'available' OR state = 'completed'"), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterNull: `("finalized_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id) OR "finalized_at" IS NULL)`, + wantAfterNull: `(finalized_at IS NULL AND id > @after_id)`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id) OR finalized_at IS NULL)`, }, { name: "FinalizedStates", params: NewJobListParams().OrderBy(JobListOrderByFinalizedAt, SortOrderAsc), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id))`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id))`, }, { name: "ScheduledAt", params: NewJobListParams().States(rivertype.JobStateAvailable, rivertype.JobStateCompleted).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "scheduled_at ASC NULLS LAST, id ASC", - wantAfterTime: `("scheduled_at" > @cursor_time OR ("scheduled_at" = @cursor_time AND "id" > @after_id))`, + wantAfterTime: `(scheduled_at > @cursor_time OR (scheduled_at = @cursor_time AND id > @after_id))`, }, } { t.Run(tt.name, func(t *testing.T) { diff --git a/js/CHANGELOG.md b/js/CHANGELOG.md index 9d8e2d797..4752fe72e 100644 --- a/js/CHANGELOG.md +++ b/js/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `@riverqueue/driver-mysql` with worker execution, transactional job operations, outbox notifications, maintenance, and API/CLI migrations. + - Support Node.js 24 across all packages. River automatically installs a `Temporal` polyfill when the runtime has no native implementation, preserving native `Temporal` on Node.js 26. ### Changed diff --git a/js/README.md b/js/README.md index 3e0f35037..147f3e392 100644 --- a/js/README.md +++ b/js/README.md @@ -1,6 +1,6 @@ # River for JavaScript and TypeScript -River is a fast, reliable background job system backed by Postgres or SQLite. +River is a fast, reliable background job system backed by MySQL, Postgres, or SQLite. This implementation runs on Node.js and shares River's database protocol with River's other language implementations, so services can insert and work the same jobs in the same database. @@ -85,13 +85,15 @@ River never closes the pool you give it and never migrates on its own. Web servers that only insert jobs construct the same `Client` without `queues` or `workers` and never call `start()`. +For MySQL, use [`@riverqueue/driver-mysql`](./driver/mysql/README.md) with `mysql2`. + For SQLite, install `@riverqueue/driver-sqlite` instead of the Postgres packages; it uses Node's built-in `node:sqlite`. See the [SQLite driver](./driver/sqlite/README.md). ## Transactions and other features -[Insert jobs in an application transaction](./docs/README.md#transactions) so the job and its related database changes commit or roll back together. River supports transactions with node-postgres, Prisma, and SQLite. +[Insert jobs in an application transaction](./docs/README.md#transactions) so the job and its related database changes commit or roll back together. River supports transactions with mysql2, node-postgres, Prisma, and SQLite. - [Bulk insertion](./docs/README.md#batches) and [unique jobs](./docs/README.md#unique-jobs). - [Retries, timeouts, cancellation, and snoozing](./docs/errors-and-retries.md). @@ -105,10 +107,11 @@ packages; it uses Node's built-in `node:sqlite`. See the | Package | Purpose | | ---------------------------- | ------------------------------------------------------------------- | | `riverqueue` | Job definitions, insertion, workers, runtime, queries, and events | +| `@riverqueue/driver-mysql` | MySQL through `mysql2` | | `@riverqueue/driver-pg` | Postgres through `node-postgres` | | `@riverqueue/driver-prisma` | Insert jobs inside Prisma transactions | | `@riverqueue/driver-sqlite` | SQLite through Node's built-in `node:sqlite` | -| `@riverqueue/migrate` | Postgres and SQLite migrations | +| `@riverqueue/migrate` | MySQL, Postgres, and SQLite migrations | | `@riverqueue/worker-threads` | Run CPU-bound handlers on worker threads | | `@riverqueue/test` | Test helpers for producers and workers | | `@riverqueue/cli` | The `riverqueue` command: migrations, benchmarks, and a 0.1 codemod | diff --git a/js/cli/package.json b/js/cli/package.json index 96915fcfc..3f424168d 100644 --- a/js/cli/package.json +++ b/js/cli/package.json @@ -52,7 +52,8 @@ "@riverqueue/driver-pg": "workspace:0.2.0", "@riverqueue/migrate": "workspace:0.2.0", "pg": "^8.22.0", - "riverqueue": "workspace:0.2.0" + "riverqueue": "workspace:0.2.0", + "mysql2": "^3.24.4" }, "peerDependencies": { "@types/node": ">=24", diff --git a/js/cli/src/database.ts b/js/cli/src/database.ts index 942ce05df..5b17a1742 100644 --- a/js/cli/src/database.ts +++ b/js/cli/src/database.ts @@ -16,6 +16,7 @@ export type DatabaseLocation = readonly backend: "postgres"; readonly connectionString: string | undefined; } + | { readonly backend: "mysql"; readonly connectionString: string } | { readonly backend: "sqlite"; readonly path: string | URL }; // Match River's Go CLI: bound statements so a stuck lock can't hang a @@ -65,6 +66,8 @@ export function parseDatabaseUrl( const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(url)?.[1]?.toLowerCase(); switch (scheme) { + case "mysql": + return { backend: "mysql", connectionString: url }; case "postgres": case "postgresql": return { backend: "postgres", connectionString: url }; @@ -83,7 +86,7 @@ export function parseDatabaseUrl( } default: throw new UsageError( - "--database-url must start with postgres://, postgresql://, or sqlite://", + "--database-url must start with postgres://, postgresql://, mysql://, or sqlite://", command ); } diff --git a/js/cli/src/migrate-commands.ts b/js/cli/src/migrate-commands.ts index 9d2513a42..a2ceaeeb4 100644 --- a/js/cli/src/migrate-commands.ts +++ b/js/cli/src/migrate-commands.ts @@ -1,3 +1,4 @@ +import { createPool } from "mysql2/promise"; import { quoteIdentifier } from "riverqueue/unstable-driver"; import { createMigrator, @@ -33,7 +34,7 @@ const TEMPLATE_SCHEMA = "/* TEMPLATE: schema */"; const DATABASE_URL_OPTION: OptionSpec = { description: - "Database to use: postgres://..., postgresql://..., or sqlite://PATH " + + "Database to use: mysql://..., postgres://..., postgresql://..., or sqlite://PATH " + "(defaults to PG* environment variables when PGDATABASE is set)", type: "string", valueName: "URL", @@ -47,7 +48,7 @@ const LINE_OPTION: OptionSpec = { const SCHEMA_OPTION: OptionSpec = { description: - "Postgres schema containing River's tables (default: the search_path)", + "MySQL database or Postgres schema containing River's tables (default: the connection's current schema)", type: "string", valueName: "NAME", }; @@ -319,7 +320,18 @@ async function runMigrateGet( }); const direction = down ? "down" : "up"; - const prefix = schema === undefined ? "" : `${quoteIdentifier(schema)}.`; + if ( + backend === "mysql" && + schema !== undefined && + !/^[A-Za-z_][A-Za-z0-9_]{0,45}$/.test(schema) + ) + throw new UsageError("Invalid MySQL schema name", command); + const prefix = + schema === undefined + ? "" + : backend === "mysql" + ? `\`${schema}\`.` + : `${quoteIdentifier(schema)}.`; let printed = false; for (const migration of selected) { if (excluded.has(migration.version)) continue; @@ -373,6 +385,25 @@ async function withMigrator( } } + if (location.backend === "mysql") { + if (statementTimeoutMs !== undefined) + throw new UsageError( + "--statement-timeout only applies to Postgres", + command + ); + const pool = createPool(location.connectionString); + try { + return await run( + createMigrator( + { backend: "mysql", pool, schema }, + { line, migrations: lineMigrations(context, line, "mysql") } + ) + ); + } finally { + await pool.end(); + } + } + const pool = openPostgresPool(location, { max: 2, statementTimeoutMs }); try { return await run( @@ -459,9 +490,10 @@ function sqlDialect( if (url === undefined) return "postgres"; const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(url)?.[1]?.toLowerCase(); if (scheme === "postgres" || scheme === "postgresql") return "postgres"; + if (scheme === "mysql") return "mysql"; if (scheme === "sqlite") return "sqlite"; throw new UsageError( - "--database-url must start with postgres://, postgresql://, or sqlite://", + "--database-url must start with postgres://, postgresql://, mysql://, or sqlite://", command ); } diff --git a/js/cli/src/run.test.ts b/js/cli/src/run.test.ts index ffff46908..5ae16b532 100644 --- a/js/cli/src/run.test.ts +++ b/js/cli/src/run.test.ts @@ -163,7 +163,7 @@ describe("argument errors", () => { it.each([ [ - ["migrate-up", "--database-url", "mysql://localhost/river"], + ["migrate-up", "--database-url", "oracle://localhost/river"], "--database-url must start with postgres://", ], [ diff --git a/js/docs/development.md b/js/docs/development.md index eb9a28c44..c5a29aca9 100644 --- a/js/docs/development.md +++ b/js/docs/development.md @@ -206,6 +206,14 @@ both drivers' notification adapters without a Postgres server. The two raw JSON unique-key cases involving duplicate keys or integer-key insertion order are Rust-only because JavaScript objects cannot preserve them. +`TEST_MYSQL_URL=mysql://root@localhost/ make test/js/conformance/mysql` runs +the MySQL storage checks against the same Go fixtures: state masks and +uniqueness conflicts, attempt errors, rescue/output metadata, exact snooze +counters, and notification emission and reception. Each test owns a temporary +database created with the canonical migrations. The shared Conformance workflow +runs both fixture targets when Go changes; the MySQL integration jobs include +these cases too. + ## Preparing a release Run this section's commands from the repository root. The eight publishable packages are `js/package.json` (`riverqueue`), the three drivers under `js/driver/*`, and `js/{migrate,worker-threads,test,cli}`. They share one version, independently of Go and Rust. Examples are private and stay at `0.0.0`. `VERSION` has no leading `v`; JavaScript Git tags use `js/vX.Y.Z`. diff --git a/js/driver/mysql/README.md b/js/driver/mysql/README.md new file mode 100644 index 000000000..12ccc00ac --- /dev/null +++ b/js/driver/mysql/README.md @@ -0,0 +1,53 @@ +# MySQL driver + +`@riverqueue/driver-mysql` provides River's client, worker runtime, and migrations +on MySQL 8.0 and later. Use the latest patch release of your MySQL version. +Jobs use the same schema and protocol as River for Go. + +```sh +npm install riverqueue @riverqueue/driver-mysql @riverqueue/migrate mysql2 +``` + +```ts +import { createPool } from "mysql2/promise"; +import { Client, defineJob, Workers } from "riverqueue"; +import { MySqlDriver } from "@riverqueue/driver-mysql"; +import { createMigrator } from "@riverqueue/migrate"; + +const pool = createPool("mysql://localhost/my_app"); +const driver = new MySqlDriver(pool); +await createMigrator(driver).migrateUp(); + +const message = defineJob({ kind: "message" }); +const workers = new Workers().add(message, () => + console.log("Message received") +); +const client = new Client(driver, { + queues: { default: { maxWorkers: 10 } }, + workers, +}); +const run = await client.start(); +await client.insert(message, {}); +await run.stop(); +await pool.end(); +``` + +Run migrations during deployment before starting workers. The CLI accepts +`--database-url mysql://localhost/my_app`. MySQL DDL commits implicitly: a +migration failure can leave partially applied DDL, while migration history is +recorded only after a version succeeds. Inspect and repair the failed version +before retrying. A named lock serializes migrators for the same database. + +Pass `{ schema: "background_jobs" }` to the driver to keep River's tables in a +separate MySQL database. River never closes your pool. Its own transactions use +READ COMMITTED; caller transactions retain their configured isolation level. + +To insert within an application transaction, check out a connection, call +`beginTransaction()`, and pass it as `{ tx }` to River. Commit or roll back the +connection yourself, then release it. Worker contexts also support completing +jobs in that transaction through `completeTx`. + +Claims use `FOR UPDATE SKIP LOCKED`. Notifications use the durable +`river_notification` outbox, including cancellation, queue controls, and +leadership resignation requests. Job IDs remain `bigint` and timestamps retain +microsecond precision regardless of the pool's JSON, date, or bigint options. diff --git a/js/driver/mysql/etc/driver-mysql.api.md b/js/driver/mysql/etc/driver-mysql.api.md new file mode 100644 index 000000000..c36f922fb --- /dev/null +++ b/js/driver/mysql/etc/driver-mysql.api.md @@ -0,0 +1,33 @@ +# API report: `@riverqueue/driver-mysql` + + + +This report contains declarations and TSDoc for names exported by the +package entry point. Private members, unexported implementation +declarations, and file layout are omitted. + +## `MySqlDriver` + +```ts +/** + * MySQL 8.0.17+ backend using a caller-owned mysql2 pool. Pass an open + * connection as `{ tx }` to join an application transaction. River never + * closes the pool or commits a caller's transaction. + */ +export declare class MySqlDriver { + readonly "~river"?: { + readonly capability: "runtime"; + readonly transaction: PoolConnection; + }; + constructor(pool: Pool, options?: MySqlDriverOptions); +} +``` + +## `MySqlDriverOptions` + +```ts +/** MySQL database containing River's tables; defaults to the pool's database. */ +export interface MySqlDriverOptions { + readonly schema?: string; +} +``` diff --git a/js/driver/mysql/package.json b/js/driver/mysql/package.json new file mode 100644 index 000000000..7707c7b03 --- /dev/null +++ b/js/driver/mysql/package.json @@ -0,0 +1,60 @@ +{ + "name": "@riverqueue/driver-mysql", + "version": "0.2.0", + "description": "MySQL backend for River's TypeScript runtime.", + "type": "module", + "sideEffects": false, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "engines": { + "node": ">=24" + }, + "files": [ + "dist", + "src", + "!src/**/*.test.ts", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "node ../../node_modules/typescript/bin/tsc && node ../../scripts/copy-license.mjs", + "clean": "rm -rf dist", + "prepack": "pnpm run clean && pnpm run build" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/riverqueue/river.git", + "directory": "js/driver/mysql" + }, + "contributors": [ + "Brandur Leach", + "Blake Gentry" + ], + "license": "MPL-2.0", + "publishConfig": { + "access": "public", + "provenance": true + }, + "peerDependencies": { + "mysql2": ">=3.0.0", + "riverqueue": "workspace:0.2.0" + }, + "devDependencies": { + "mysql2": "^3.24.4", + "riverqueue": "workspace:0.2.0", + "typescript": "^6.0.3" + }, + "keywords": [ + "river", + "job-queue", + "mysql", + "driver" + ] +} diff --git a/js/driver/mysql/src/codecs.ts b/js/driver/mysql/src/codecs.ts new file mode 100644 index 000000000..a04613356 --- /dev/null +++ b/js/driver/mysql/src/codecs.ts @@ -0,0 +1,118 @@ +import { parseJson, stringifyJson, toJsonObject } from "riverqueue"; +import type { JobRow } from "riverqueue"; +import { + decodeAttemptErrors, + decodeJobState, + recordQueueMetadataText, + uniqueBitmaskToStates, +} from "riverqueue/unstable-driver"; +import type { QueueRow } from "riverqueue/unstable-driver"; + +export type Row = Record; + +// Cast at the SQL boundary so caller pool options cannot truncate IDs, JSON +// numbers, or microseconds, or interpret UTC DATETIME values in local time. +export const JOB_COLUMNS = + `CAST(id AS CHAR) AS id, args, attempt, attempted_at, attempted_by, +created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, +scheduled_at, state, tags, unique_key, unique_states`.replace( + /\b(args|attempted_by|errors|metadata|tags|attempted_at|created_at|finalized_at|scheduled_at)\b/g, + "CAST($1 AS CHAR) AS $1" + ); +export const QUEUE_COLUMNS = `name, CAST(created_at AS CHAR) AS created_at, +CAST(metadata AS CHAR) AS metadata, CAST(paused_at AS CHAR) AS paused_at, CAST(updated_at AS CHAR) AS updated_at`; + +export function timestamp( + value: Temporal.Instant | null | undefined +): string | null { + return value == null + ? null + : value + .toString({ fractionalSecondDigits: 6 }) + .replace("T", " ") + .replace(/Z$/, ""); +} + +export function instant(value: unknown): Temporal.Instant { + if (typeof value !== "string") + throw new TypeError("MySQL timestamp is not text"); + return Temporal.Instant.from(`${value.replace(" ", "T")}Z`); +} + +function strings(value: unknown): string[] { + if (value === null) return []; + const array = parseJson(text(value)); + if (array === null) return []; + if (!Array.isArray(array) || array.some((item) => typeof item !== "string")) + throw new TypeError("Expected a JSON array of strings"); + return array as string[]; +} + +export function decodeJob(row: Row): JobRow { + const result = decodeJobPartial(row); + if (result.error) throw result.error; + return result.job; +} + +export function decodeJobPartial(row: Row): { job: JobRow; error?: Error } { + const errors: Error[] = []; + const decode = (fallback: T, callback: () => T): T => { + try { + return callback(); + } catch (error) { + errors.push(error instanceof Error ? error : new Error(String(error))); + return fallback; + } + }; + const job: JobRow = { + args: decode({}, () => toJsonObject(parseJson(String(row.args)))), + attempt: Math.max(0, Number(row.attempt)), + attemptedAt: row.attempted_at === null ? null : instant(row.attempted_at), + attemptedBy: decode([], () => strings(row.attempted_by)), + createdAt: instant(row.created_at), + errors: decode([], () => + row.errors === null ? [] : decodeAttemptErrors(text(row.errors)) + ), + finalizedAt: row.finalized_at === null ? null : instant(row.finalized_at), + id: BigInt(String(row.id)), + kind: String(row.kind), + maxAttempts: Math.max(0, Number(row.max_attempts)), + metadata: decode({}, () => toJsonObject(parseJson(String(row.metadata)))), + priority: Number(row.priority), + queue: String(row.queue), + scheduledAt: instant(row.scheduled_at), + state: decodeJobState(String(row.state)), + tags: decode([], () => strings(row.tags)), + uniqueKey: + row.unique_key === null + ? null + : new Uint8Array(row.unique_key as Uint8Array), + uniqueStates: + row.unique_states === null + ? null + : uniqueBitmaskToStates(Number(row.unique_states)), + }; + return errors.length + ? { job, error: new AggregateError(errors, "Invalid MySQL job row") } + : { job }; +} + +export function decodeQueue(row: Row): QueueRow { + const queue = { + createdAt: instant(row.created_at), + metadata: toJsonObject(parseJson(String(row.metadata))), + name: String(row.name), + pausedAt: row.paused_at === null ? null : instant(row.paused_at), + updatedAt: instant(row.updated_at), + }; + recordQueueMetadataText(queue, String(row.metadata)); + return queue; +} + +export { stringifyJson as json }; + +export function text(value: unknown): string { + if (typeof value !== "string") + throw new TypeError("Expected MySQL text column"); + return value; +} diff --git a/js/driver/mysql/src/driver.integration.test.ts b/js/driver/mysql/src/driver.integration.test.ts new file mode 100644 index 000000000..642c9b4a0 --- /dev/null +++ b/js/driver/mysql/src/driver.integration.test.ts @@ -0,0 +1,833 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { createPool, type Pool, type RowDataPacket } from "mysql2/promise"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + Client, + defineJob, + Workers, + parseJson, + isExactJsonNumber, + jsonNumberToBigInt, + stringifyJson, + type ExactJsonNumber, + type JobState, + type JsonObject, +} from "riverqueue"; +import type { + JobInsertParams, + JobListParams, + JobCompletionCommand, +} from "riverqueue/unstable-driver"; +import { createMigrator } from "../../../migrate/src/index.js"; +import { decodeAttemptError } from "../../../src/driver-codecs.js"; +import { completionCommand } from "../../../src/runtime/completion-command.js"; +import { MySqlDriver, MySqlRuntime } from "./driver.js"; + +function fixture(name: string) { + return parseJson( + readFileSync( + new URL(`../../../../conformance/testdata/${name}.json`, import.meta.url), + "utf8" + ) + ); +} + +const url = process.env.TEST_MYSQL_URL; +const listing: JobListParams = { + after: null, + ids: [], + kinds: [], + limit: 100, + metadata: null, + priorities: [], + queues: [], + sortDirection: "asc", + sortField: "id", + states: [], + tagsAll: [], + tagsAny: [], +}; +function params(overrides: Partial = {}): JobInsertParams { + return { + args: { message: "hello" }, + encodedArgs: '{"message":"hello"}', + kind: "message", + maxAttempts: 25, + metadata: {}, + priority: 1, + queue: "default", + // Use the claim clock so small client/server clock differences cannot defer a job. + scheduledAt: Temporal.Now.instant(), + state: "available", + tags: [], + uniqueKey: null, + uniqueStates: null, + ...overrides, + }; +} + +describe.skipIf(!url)("MySQL driver", () => { + let pool: Pool; + let schema: string; + let driver: MySqlDriver; + let runtime: MySqlRuntime; + beforeEach(async () => { + pool = createPool({ uri: url!, connectionLimit: 12 }); + schema = `js_${randomUUID().replaceAll("-", "")}`; + driver = new MySqlDriver(pool, { schema }); + runtime = new MySqlRuntime(pool, { schema }); + await createMigrator(driver).migrateUp(); + }); + afterEach(async () => { + await pool.query(`DROP DATABASE IF EXISTS \`${schema}\``); + await pool.end(); + }); + + it("migrates down and back up", async () => { + const migrator = createMigrator(driver); + expect((await migrator.validate()).ok).toBe(true); + await migrator.migrateDown({ targetVersion: 0 }); + expect(await migrator.existingVersions()).toEqual([]); + await migrator.migrateUp(); + expect((await migrator.validate()).ok).toBe(true); + }); + + it("preserves large IDs, JSON integers, microseconds, and time zones", async () => { + const time = Temporal.Instant.from("2026-10-01T02:03:04.123456Z"); + await pool.query( + `ALTER TABLE \`${schema}\`.river_job AUTO_INCREMENT=9007199254740993` + ); + const first = await runtime.jobInsert( + params({ + encodedArgs: '{"n":9007199254740993}', + args: parseJson('{"n":9007199254740993}') as { n: number }, + createdAt: time, + scheduledAt: time, + }) + ); + const second = await runtime.jobInsert(params()); + expect(first.job.id).toBe(9007199254740993n); + expect(second.job.id).toBe(first.job.id + 1n); + expect(isExactJsonNumber(first.job.args.n)).toBe(true); + expect(first.job.createdAt.equals(time)).toBe(true); + expect(first.job.scheduledAt.equals(time)).toBe(true); + expect( + (await runtime.jobList({ ...listing, ids: [second.job.id] })).map( + (job) => job.id + ) + ).toEqual([second.job.id]); + }); + + it("joins caller transactions and keeps rollback atomic", async () => { + const tx = await pool.getConnection(); + try { + await expect(runtime.jobInsert(params(), { tx })).rejects.toMatchObject({ + code: "transaction_scope", + }); + await tx.beginTransaction(); + const inserted = await runtime.jobInsert(params(), { tx }); + await runtime.notifyInsert(["default"], { tx }); + expect(await runtime.jobGet(inserted.job.id)).toBeNull(); + await tx.rollback(); + expect(await runtime.jobGet(inserted.job.id)).toBeNull(); + const [rows] = await pool.query( + `SELECT * FROM \`${schema}\`.river_notification` + ); + expect(rows).toEqual([]); + } finally { + tx.release(); + } + }); + + it("deduplicates concurrent inserts without changing the existing kind", async () => { + const uniqueKey = new Uint8Array(32).fill(3); + const inserts = await Promise.all( + Array.from({ length: 8 }, () => + runtime.jobInsert(params({ uniqueKey, uniqueStates: ["available"] })) + ) + ); + expect(inserts.filter((item) => item.status === "inserted")).toHaveLength( + 1 + ); + expect(new Set(inserts.map((item) => item.job.id)).size).toBe(1); + const duplicate = await runtime.jobInsert( + params({ kind: "different", uniqueKey, uniqueStates: ["available"] }) + ); + expect(duplicate.status).toBe("duplicate"); + expect(duplicate.job.kind).toBe("message"); + }); + + it("allows repeated batch keys outside the unique states", async () => { + const uniqueKey = new Uint8Array(32).fill(4); + const pending = params({ + uniqueKey, + uniqueStates: ["available"], + state: "pending", + }); + const active = params({ uniqueKey, uniqueStates: ["available"] }); + const inserted = await runtime.jobInsertMany([pending, active, pending]); + expect(inserted.map((item) => item.status)).toEqual([ + "inserted", + "inserted", + "inserted", + ]); + expect(new Set(inserted.map((item) => item.job.id)).size).toBe(3); + await expect( + runtime.jobInsertMany([active, pending, active]) + ).rejects.toThrow("Batch repeats a unique key"); + expect(await runtime.jobList(listing)).toHaveLength(3); + }); + + it("rejects unrelated unique constraints when the incoming key is inactive", async () => { + await pool.query( + `CREATE UNIQUE INDEX custom_kind ON \`${schema}\`.river_job(kind)` + ); + const uniqueKey = new Uint8Array(32).fill(5); + const original = await runtime.jobInsert( + params({ uniqueKey, uniqueStates: ["available"] }) + ); + await expect( + runtime.jobInsert( + params({ uniqueKey, uniqueStates: ["available"], state: "pending" }) + ) + ).rejects.toMatchObject({ + cause: { message: "Insertion violated an unrelated unique constraint" }, + }); + expect(await runtime.jobGet(original.job.id)).toEqual(original.job); + expect(await runtime.jobList(listing)).toHaveLength(1); + }); + + it("claims concurrently without duplicate work, and respects paused queues", async () => { + await runtime.runtimeQueueUpsert("paused", Temporal.Now.instant()); + await runtime.queuePause("paused"); + await runtime.jobInsert(params({ queue: "paused" })); + await runtime.jobInsertMany(Array.from({ length: 20 }, () => params())); + const batches = await Promise.all( + Array.from({ length: 4 }, (_, index) => + runtime.jobClaim({ + attemptedBy: `worker${index}`, + kinds: [], + queues: [ + { name: "default", limit: 10 }, + { name: "paused", limit: 1 }, + ], + }) + ) + ); + const ids = batches.flatMap((batch) => batch.jobs.map((job) => job.id)); + // A peer may skip rows while the first claimant holds secondary-index + // locks. The next poll must claim the remainder, with no duplicate IDs. + const remaining = await runtime.jobClaim({ + attemptedBy: "next", + kinds: [], + queues: [{ name: "default", limit: 20 }], + }); + ids.push(...remaining.jobs.map((job) => job.id)); + expect(ids).toHaveLength(20); + expect(new Set(ids).size).toBe(20); + await runtime.queueResume("paused"); + expect( + ( + await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "paused", limit: 1 }], + }) + ).jobs + ).toHaveLength(1); + }); + + it("fences completions and makes a cancellation win over a retry", async () => { + const inserted = await runtime.jobInsert( + params({ metadata: { keep: true, nested: { a: 1 } } }) + ); + const claimed = await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const job = claimed.jobs[0]!; + const command: JobCompletionCommand = { + id: job.id, + attempt: job.attempt, + attemptedBy: "worker", + kind: "retry", + finalizedAt: null, + scheduledAt: Temporal.Now.instant().add({ hours: 1 }), + error: { at: Temporal.Now.instant(), error: "retry", trace: "" }, + metadata: { nested: { b: 2 }, present: null }, + output: null, + outputSet: true, + }; + expect( + (await runtime.jobCompleteMany([{ ...command, attempt: 999 }]))[0]?.status + ).toBe("stale"); + await runtime.jobCancel(inserted.job.id); + const result = (await runtime.jobCompleteMany([command]))[0]!; + expect(result.status).toBe("applied"); + expect(result.job?.state).toBe("cancelled"); + expect(result.job?.metadata).toMatchObject({ + keep: true, + nested: { b: 2 }, + present: null, + output: null, + }); + expect(result.job?.metadata.nested).toEqual({ b: 2 }); + expect(result.job?.errors).toHaveLength(1); + }); + + it.each([ + [{ value: { a: 1, b: 2 } }, { value: { a: 1 } }, true], + [{ value: { a: 1, b: 2 } }, { value: { a: 2 } }, false], + [{ value: [[1]] }, { value: [1] }, false], + [{ value: [1] }, { value: 1 }, false], + [{ value: [{ a: 1, b: 2 }] }, { value: [{ b: 2 }] }, true], + [{ tenant: { settings: [] } }, { tenant: { settings: {} } }, false], + [{ tenant: { settings: { a: 1 } } }, { tenant: { settings: {} } }, true], + ])( + "preserves nested metadata containment %#", + async (metadata, wanted, matches) => { + await runtime.jobInsert(params({ metadata: { unrelated: true } })); + const inserted = await runtime.jobInsert(params({ metadata })); + const rows = await runtime.jobList({ ...listing, metadata: wanted }); + expect(rows.map((row) => row.id)).toEqual( + matches ? [inserted.job.id] : [] + ); + } + ); + + it("updates queues and jobs, retries and deletes", async () => { + await runtime.runtimeQueueUpsert("default", Temporal.Now.instant()); + expect( + (await runtime.queueUpdate("default", { metadata: { concurrency: 5 } })) + ?.metadata + ).toEqual({ concurrency: 5 }); + expect( + await runtime.queueList({ nameAfter: null, limit: 10 }) + ).toHaveLength(1); + const inserted = await runtime.jobInsert(params()); + await runtime.jobUpdate(inserted.job.id, { + metadata: { nested: { value: true } }, + output: null, + }); + expect((await runtime.jobCancel(inserted.job.id))?.state).toBe("cancelled"); + expect((await runtime.jobRetry(inserted.job.id))?.state).toBe("available"); + expect((await runtime.jobDelete(inserted.job.id)).status).toBe("deleted"); + await runtime.jobInsertMany([params(), params({ kind: "other" })]); + expect( + await runtime.jobDeleteMany({ ...listing, all: false, kinds: ["other"] }) + ).toHaveLength(1); + }); + + it("cleans expired jobs without waiting for locked jobs", async () => { + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "cleaner", + now, + 60_000, + null + ))!; + const [running, locked, ...expired] = await runtime.jobInsertMany( + Array.from({ length: 1002 }, () => params()) + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='completed', finalized_at='1970-01-01' WHERE id>=?`, + [locked!.job.id.toString()] + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='running' WHERE id=?`, + [running!.job.id.toString()] + ); + const tx = await pool.getConnection(); + try { + await tx.beginTransaction(); + for (const id of [running!.job.id, locked!.job.id]) { + await tx.query( + `SELECT id FROM \`${schema}\`.river_job WHERE id=? FOR UPDATE`, + [id.toString()] + ); + } + expect( + await runtime.maintenanceCleanJobs( + leader, + { + cancelledBefore: now, + completedBefore: now, + discardedBefore: now, + limit: 1000, + }, + null, + AbortSignal.timeout(2_000) + ) + ).toBe(1000); + expect(await runtime.jobGet(expired[0]!.job.id)).toBeNull(); + expect((await runtime.jobGet(locked!.job.id))?.state).toBe("completed"); + expect((await runtime.jobGet(running!.job.id))?.state).toBe("running"); + } finally { + await tx.rollback(); + tx.release(); + } + }); + + it("schedules, rescues and cleans under an exact leader term", async () => { + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "leader", + now, + 60_000, + null + ))!; + expect(leader).not.toBeNull(); + expect( + await runtime.maintenanceLeaderAcquire("follower", now, 60_000, null) + ).toBeNull(); + const inserted = await runtime.jobInsert( + params({ state: "scheduled", scheduledAt: now }) + ); + expect( + await runtime.maintenanceSchedule(leader, { + now, + limit: 100, + notificationHorizon: now, + scheduledAtHorizon: now, + allowInsertNotifications: (queues) => queues, + }) + ).toBe(1); + await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const horizon = Temporal.Now.instant().add({ hours: 1 }); + expect( + await runtime.maintenanceGetStuck(leader, horizon, 0n, 100) + ).toHaveLength(1); + expect( + await runtime.maintenanceRescue(leader, horizon, [ + { + id: inserted.job.id, + state: "discarded", + finalizedAt: now, + scheduledAt: now, + error: { at: now, attempt: 1, error: "stuck", trace: "" }, + }, + ]) + ).toBe(1); + expect( + (await runtime.jobGet(inserted.job.id))?.metadata["river:rescue_count"] + ).toBe(1); + expect( + await runtime.maintenanceCleanJobs( + leader, + { + limit: 100, + cancelledBefore: horizon, + completedBefore: horizon, + discardedBefore: horizon, + }, + null, + new AbortController().signal + ) + ).toBe(1); + expect(await runtime.maintenanceLeaderResign(leader)).toBe(true); + expect( + await runtime.maintenanceSchedule(leader, { + now, + limit: 100, + notificationHorizon: now, + scheduledAtHorizon: now, + allowInsertNotifications: (queues) => queues, + }) + ).toBe(0); + }); + + it("delivers request_resign and waits for earlier uncommitted notifications", async () => { + const abort = new AbortController(); + const messages: string[] = []; + const ready = Promise.withResolvers(); + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["leadership", "insert"], + abort.signal, + () => ready.resolve(undefined) + )) + messages.push(message.payload); + })(); + await ready.promise; + const first = await pool.getConnection(); + try { + await first.beginTransaction(); + await runtime.notifyInsert(["first"], { tx: first }); + await runtime.notifyInsert(["second"]); + await first.commit(); + await runtime.runtimeRequestLeadershipResignation(); + await expect.poll(() => messages.length).toBe(3); + expect(messages.map((message) => JSON.parse(message))).toEqual([ + { queue: "first" }, + { queue: "second" }, + { action: "request_resign", leader_id: "" }, + ]); + } finally { + first.release(); + abort.abort(); + await listening; + } + }); + + it("interrupts a notification poll blocked on an uncommitted row", async () => { + const abort = new AbortController(); + const ready = Promise.withResolvers(); + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["insert"], + abort.signal, + () => ready.resolve(undefined) + )) + expect(message).toBeUndefined(); + })(); + await ready.promise; + const first = await pool.getConnection(); + try { + await first.beginTransaction(); + await runtime.notifyInsert(["blocked"], { tx: first }); + await expect + .poll(async () => { + const [rows] = await pool.query< + import("mysql2/promise").RowDataPacket[] + >( + "SELECT COUNT(*) AS n FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'", + [schema] + ); + return Number(rows[0]?.n); + }) + .toBeGreaterThan(0); + abort.abort(); + await listening; + expect(await runtime.jobList(listing)).toEqual([]); + } finally { + abort.abort(); + await first.rollback(); + first.release(); + await listening; + } + }); + + it("matches job kinds and queues exactly, including trailing spaces", async () => { + const wanted = (await runtime.jobInsert(params())).job; + await runtime.jobInsert(params({ kind: "message ", queue: "default " })); + expect( + (await runtime.jobList({ ...listing, kinds: ["message"] })).map( + (job) => job.id + ) + ).toEqual([wanted.id]); + expect( + (await runtime.jobList({ ...listing, queues: ["default"] })).map( + (job) => job.id + ) + ).toEqual([wanted.id]); + }); + + it("runs the public worker API to completion", async () => { + const message = defineJob({ kind: "message" }); + const workers = new Workers().add(message, () => undefined); + const client = new Client(driver, { + queues: { default: { maxWorkers: 2 } }, + workers, + completionFlushInterval: { milliseconds: 1 }, + }); + const run = await client.start(); + try { + const inserted = await client.insert(message, {}); + await expect + .poll(async () => (await client.jobs.get(inserted.job.id))?.state, { + timeout: 5000, + }) + .toBe("completed"); + } finally { + await run.stop(); + } + }); + + describe("Go conformance", () => { + if (!url) return; + + const protocol = fixture("protocol_values") as unknown as { + attempt_error: JsonObject; + job_states: { state: JobState; unique_bit: number }[]; + metadata_keys: { + output: string; + rescue_count: string; + unique_nonce: string; + }; + notifications: { name: string; topic: string; payload: JsonObject }[]; + }; + const snoozes = fixture("snooze_counters") as unknown as { + snooze_counters: { + name: string; + metadata: JsonObject; + expected_snoozes: ExactJsonNumber | number; + }[]; + }; + + it.each(protocol.job_states)( + "stores $state and its unique bit", + async ({ state, unique_bit: bit }) => { + const inserted = await runtime.jobInsert( + params({ + uniqueKey: new Uint8Array(32).fill(1), + uniqueStates: [state], + }) + ); + const [encoded] = await pool.query( + `SELECT unique_states FROM \`${schema}\`.river_job WHERE id=?`, + [inserted.job.id.toString()] + ); + expect(encoded[0]?.unique_states).toBe(bit); + const finalized = ["cancelled", "completed", "discarded"].includes( + state + ); + // Seed the Go state/mask independently of the driver's encoder. + await pool.query( + `UPDATE \`${schema}\`.river_job SET state=?, finalized_at=${finalized ? "UTC_TIMESTAMP(6)" : "NULL"}, unique_states=? WHERE id=?`, + [state, bit, inserted.job.id.toString()] + ); + const stored = await runtime.jobGet(inserted.job.id); + expect(stored?.state).toBe(state); + expect(stored?.uniqueStates).toEqual([state]); + expect( + await runtime.jobInsert( + params({ + uniqueKey: inserted.job.uniqueKey, + uniqueStates: ["available", state], + }) + ) + ).toMatchObject({ status: "duplicate", job: { id: inserted.job.id } }); + const [rows] = await pool.query( + `SELECT unique_states FROM \`${schema}\`.river_job WHERE id=?`, + [inserted.job.id.toString()] + ); + expect(rows[0]?.unique_states).toBe(bit); + } + ); + + it.each(snoozes.snooze_counters)( + "persists the $name snooze counter", + async ({ metadata, expected_snoozes: expected }) => { + const inserted = await runtime.jobInsert(params({ metadata })); + const { jobs } = await runtime.jobClaim({ + attemptedBy: "client", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const now = Temporal.Now.instant(); + const command = completionCommand( + jobs[0]!, + "client", + { + status: "succeeded", + outcome: { + type: "snooze", + duration: Temporal.Duration.from({ hours: 1 }), + }, + }, + now, + now, + now, + 1000 + ); + expect((await runtime.jobCompleteMany([command]))[0]?.status).toBe( + "applied" + ); + const stored = (await runtime.jobGet(inserted.job.id))!; + expect(stored.state).toBe("scheduled"); + expect(stored.attempt).toBe(0); + expect( + jsonNumberToBigInt( + stored.metadata.snoozes as ExactJsonNumber | number + ) + ).toBe(jsonNumberToBigInt(expected)); + } + ); + + it.each([ + { initial: 2, expected: 3 }, + { initial: 2.5, expected: 3.5 }, + { initial: null, expected: 1 }, + { initial: "2", expected: 1 }, + ])( + "round trips Go's attempt error and persists rescue metadata from $initial", + async ({ initial, expected }) => { + const keys = protocol.metadata_keys; + const inserted = await runtime.jobInsert( + params({ metadata: { [keys.rescue_count]: initial } }) + ); + const nonce = inserted.job.metadata[keys.unique_nonce]; + expect(nonce).toEqual(expect.any(String)); + expect(nonce).not.toBe(""); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='running', attempt=3, attempted_at=UTC_TIMESTAMP(6), errors=JSON_ARRAY(CAST(? AS JSON)) WHERE id=?`, + [stringifyJson(protocol.attempt_error), inserted.job.id.toString()] + ); + const error = decodeAttemptError(stringifyJson(protocol.attempt_error)); + expect((await runtime.jobGet(inserted.job.id))?.errors).toEqual([ + error, + ]); + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "client", + now, + 60_000, + null + ))!; + expect( + await runtime.maintenanceRescue(leader, now.add({ hours: 1 }), [ + { + id: inserted.job.id, + state: "retryable", + scheduledAt: now, + finalizedAt: null, + error, + }, + ]) + ).toBe(1); + await runtime.maintenanceSchedule(leader, { + now, + limit: 1, + scheduledAtHorizon: now, + notificationHorizon: now, + allowInsertNotifications: (queues) => queues, + }); + const { jobs } = await runtime.jobClaim({ + attemptedBy: "client", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const command = completionCommand( + jobs[0]!, + "client", + { + status: "succeeded", + outcome: { type: "complete", output: { ok: true } }, + }, + now, + now, + now, + 1000 + ); + expect((await runtime.jobCompleteMany([command]))[0]?.status).toBe( + "applied" + ); + const stored = (await runtime.jobGet(inserted.job.id))!; + expect(stored.metadata).toMatchObject({ + [keys.output]: { ok: true }, + [keys.rescue_count]: expected, + [keys.unique_nonce]: nonce, + }); + expect(stored.errors).toEqual([error, error]); + } + ); + + it.each(protocol.notifications)( + "emits Go's $name topic and payload", + async ({ name, topic, payload }) => { + // Inputs deliberately use API semantics, not the fixture's wire keys. + switch (name) { + case "cancel": { + const inserted = await runtime.jobInsert( + params({ queue: "priority" }) + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET id=42, state='running' WHERE id=?`, + [inserted.job.id.toString()] + ); + await runtime.jobCancel(42n); + break; + } + case "insert": + await new Client(driver).insert( + defineJob({ kind: "notify" }), + {}, + { queue: "priority" } + ); + break; + case "metadata_changed": + case "pause": + case "resume": + await runtime.runtimeQueueUpsert( + "priority", + Temporal.Now.instant() + ); + if (name === "metadata_changed") + await runtime.queueUpdate("priority", { + metadata: { owner: "candidate" }, + }); + else { + await runtime.queuePause("priority"); + if (name === "resume") await runtime.queueResume("priority"); + } + break; + case "request_resign": + await runtime.runtimeRequestLeadershipResignation(); + break; + case "resigned": { + const leader = (await runtime.maintenanceLeaderAcquire( + "client-1", + Temporal.Now.instant(), + 60_000, + null + ))!; + expect(await runtime.maintenanceLeaderResign(leader)).toBe(true); + break; + } + default: + throw new Error(`Unhandled Go notification: ${name}`); + } + const [rows] = await pool.query( + `SELECT topic, CAST(payload AS CHAR) AS payload FROM \`${schema}\`.river_notification ORDER BY id DESC LIMIT 1` + ); + expect(rows[0]?.topic).toBe(topic); + expect(parseJson(rows[0]?.payload as string)).toEqual(payload); + } + ); + + it("receives Go's notifications from the MySQL outbox", async () => { + const abort = new AbortController(); + const ready = Promise.withResolvers(); + const received: { topic: string; payload: JsonObject }[] = []; + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["control", "insert", "leadership"], + abort.signal, + () => ready.resolve(undefined) + )) { + received.push({ + topic: message.topic, + payload: parseJson(message.payload) as JsonObject, + }); + } + })(); + try { + await ready.promise; + for (const { topic, payload } of protocol.notifications) { + await pool.query( + `INSERT INTO \`${schema}\`.river_notification(topic,payload) VALUES (?,?)`, + [topic, stringifyJson(payload)] + ); + } + await expect + .poll(() => received) + .toEqual( + protocol.notifications.map(({ topic, payload }) => ({ + // The runtime exposes logical topics; the outbox uses Go's prefix. + topic: topic.replace(/^river_/, ""), + payload, + })) + ); + } finally { + abort.abort(); + await listening; + } + }); + }); +}); diff --git a/js/driver/mysql/src/driver.ts b/js/driver/mysql/src/driver.ts new file mode 100644 index 000000000..388d32052 --- /dev/null +++ b/js/driver/mysql/src/driver.ts @@ -0,0 +1,1145 @@ +import { Buffer } from "node:buffer"; +import { randomBytes } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { + Pool, + PoolConnection, + ResultSetHeader, + RowDataPacket, +} from "mysql2/promise"; +import { + ConfigurationError, + DatabaseOperationError, + RiverError, + TransactionScopeError, + isExactJsonNumber, +} from "riverqueue"; +import type { JobRow, JobState, JsonObject, JsonValue } from "riverqueue"; +import { + jobCompletionKey, + jobListKeyset, + jobListKeysetSql, + queueMetadataUpdate, + registerDriver, + uniqueBitmaskFromStates, +} from "riverqueue/unstable-driver"; +import type { + DriverRecord, + DriverInsertResult, + InsertDriverOptions, + JobClaimOptions, + JobClaimParams, + JobClaimResult, + JobCompletionCommand, + JobCompletionResult, + JobDeleteManyParams, + JobDeleteResult, + JobInsertParams, + JobListParams, + JobUpdateParams, + QueueListParams, + QueueRow, + QueueUpdateParams, + RuntimeDriver, + RuntimeJobCleanupParams, + RuntimeJobRescue, + RuntimeLeader, + RuntimeMaintenanceBatch, + RuntimeNotification, + RuntimeScheduleParams, + RuntimeWaitOptions, +} from "riverqueue/unstable-driver"; +import { + decodeJob, + decodeJobPartial, + decodeQueue, + instant, + JOB_COLUMNS, + json, + QUEUE_COLUMNS, + timestamp, + text, +} from "./codecs.js"; +import type { Row } from "./codecs.js"; + +/** MySQL database containing River's tables; defaults to the pool's database. */ +export interface MySqlDriverOptions { + readonly schema?: string; +} +type Options = InsertDriverOptions & RuntimeWaitOptions; +const NONCE = "river:unique_nonce"; + +/** + * MySQL 8.0.17+ backend using a caller-owned mysql2 pool. Pass an open + * connection as `{ tx }` to join an application transaction. River never + * closes the pool or commits a caller's transaction. + */ +export class MySqlDriver { + declare readonly "~river"?: { + readonly capability: "runtime"; + readonly transaction: PoolConnection; + }; + constructor(pool: Pool, options: MySqlDriverOptions = {}) { + const runtime = new MySqlRuntime(pool, options); + registerDriver(this, runtime.driverRecord()); + } +} + +/** @internal */ +export class MySqlRuntime implements RuntimeDriver { + readonly backend = "mysql"; + readonly #pool: Pool; + readonly #prefix: string; + readonly #schema: string | undefined; + + constructor(pool: Pool, options: MySqlDriverOptions = {}) { + if (typeof pool.getConnection !== "function") + throw new ConfigurationError( + "MySqlDriver requires a mysql2 promise Pool" + ); + if ( + options.schema !== undefined && + !/^[A-Za-z_][A-Za-z0-9_]{0,45}$/.test(options.schema) + ) + throw new ConfigurationError("Invalid MySQL schema name"); + this.#pool = pool; + this.#schema = options.schema; + this.#prefix = options.schema === undefined ? "" : `\`${options.schema}\`.`; + } + + driverRecord(): DriverRecord { + return { + backend: this.backend, + capability: "runtime", + operations: this, + migration: { backend: "mysql", pool: this.#pool, schema: this.#schema }, + }; + } + + async operationScope( + tx: PoolConnection | undefined, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#transaction(tx === undefined ? {} : { tx }, callback); + } + + async #transaction( + options: Options, + callback: (tx: PoolConnection) => Promise + ): Promise { + options.signal?.throwIfAborted(); + if (options.tx !== undefined) { + if ( + typeof options.tx.execute !== "function" || + typeof options.tx.release !== "function" + ) + throw new ConfigurationError( + "Expected a mysql2 pool connection transaction" + ); + const [status] = await options.tx.query("DO 0"); + if ((status.serverStatus & 1) === 0) + throw new TransactionScopeError( + "no_transaction", + "MySQL connection has no open transaction" + ); + return callback(options.tx); + } + const acquiring = this.#pool.getConnection(); + let tx: PoolConnection; + if (options.signal) { + const signal = options.signal; + let abort!: () => void; + try { + tx = await Promise.race([ + acquiring, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + }), + ]); + } catch (error) { + void acquiring.then( + (late) => late.release(), + () => undefined + ); + throw error; + } finally { + signal.removeEventListener("abort", abort); + } + } else tx = await acquiring; + let abort: (() => void) | undefined; + try { + options.signal?.throwIfAborted(); + const operation = async () => { + await tx.query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED"); + await tx.beginTransaction(); + try { + const result = await callback(tx); + await tx.commit(); + return result; + } catch (error) { + try { + await tx.rollback(); + } catch { + tx.destroy(); + } + throw error; + } + }; + if (!options.signal) return await operation(); + const signal = options.signal; + return await Promise.race([ + new Promise((_, reject) => { + abort = () => { + // A pending locking read must not delay shutdown or return to + // the pool. Only River-owned connections are cancelled this way. + tx.destroy(); + reject(signal.reason); + }; + signal.addEventListener("abort", abort, { once: true }); + }), + operation(), + ]); + } catch (cause) { + options.signal?.throwIfAborted(); + if (cause instanceof RiverError) throw cause; + const code = (cause as { code?: string }).code ?? ""; + throw new DatabaseOperationError("MySQL operation failed", { + backend: "mysql", + operation: "transaction", + cause, + retryable: + /DEADLOCK|LOCK_WAIT_TIMEOUT|ECONN|EPIPE|ETIMEDOUT|PROTOCOL_CONNECTION_LOST/.test( + code + ), + }); + } finally { + if (abort) options.signal?.removeEventListener("abort", abort); + tx.release(); + } + } + + async #rows( + tx: PoolConnection, + sql: string, + values: readonly unknown[] = [] + ): Promise { + const [rows] = await tx.execute( + sql.replaceAll("{schema}", this.#prefix), + values.map(binding) + ); + return rows; + } + async #write( + tx: PoolConnection, + sql: string, + values: readonly unknown[] = [] + ): Promise { + const [result] = await tx.execute( + sql.replaceAll("{schema}", this.#prefix), + values.map(binding) + ); + return result.affectedRows; + } + async #get( + tx: PoolConnection, + id: bigint, + lock = false + ): Promise { + return ( + ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE id=?${lock ? " FOR UPDATE" : ""}`, + [id] + ) + )[0] ?? null + ); + } + + async jobInsert( + params: JobInsertParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const metadata = { + ...params.metadata, + [NONCE]: randomBytes(8).toString("hex"), + }; + await this.#write( + tx, + `INSERT INTO {schema}river_job (args,created_at,kind,max_attempts,metadata,priority,queue,scheduled_at,state,tags,unique_key,unique_states) + VALUES (CAST(? AS JSON),COALESCE(?,UTC_TIMESTAMP(6)),?,?,CAST(? AS JSON),?,?,COALESCE(?,UTC_TIMESTAMP(6)),?,CAST(? AS JSON),?,?) + ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id)`, + [ + params.encodedArgs, + timestamp(params.createdAt), + params.kind, + Math.min(params.maxAttempts, 2_147_483_647), + json(metadata), + params.priority, + params.queue, + timestamp(params.scheduledAt), + params.state, + json(params.tags), + params.uniqueKey === null ? null : Buffer.from(params.uniqueKey), + params.uniqueStates === null + ? null + : Number.parseInt(uniqueBitmaskFromStates(params.uniqueStates), 2), + ] + ); + const row = ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE id=LAST_INSERT_ID() FOR UPDATE` + ) + )[0]; + if (!row) throw new Error("MySQL insert returned no row"); + const job = decodeJob(row); + const duplicate = job.metadata[NONCE] !== metadata[NONCE]; + if ( + duplicate && + (params.uniqueKey === null || + !params.uniqueStates?.includes(params.state) || + job.uniqueKey === null || + !Buffer.from(params.uniqueKey).equals(job.uniqueKey) || + !job.uniqueStates?.includes(job.state)) + ) + throw new Error("Insertion violated an unrelated unique constraint"); + return { job, status: duplicate ? "duplicate" : "inserted" }; + }); + } + async jobInsertMany( + params: readonly JobInsertParams[], + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const keys = new Set(); + const results = []; + for (const item of params) { + if ( + item.uniqueKey !== null && + item.uniqueStates?.includes(item.state) + ) { + const key = Buffer.from(item.uniqueKey).toString("hex"); + if (keys.has(key)) + throw new ConfigurationError("Batch repeats a unique key"); + keys.add(key); + } + results.push(await this.jobInsert(item, { tx })); + } + return results; + }); + } + async jobGet(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id); + return row ? decodeJob(row) : null; + }); + } + async jobCancel(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return null; + const job = decodeJobPartial(row).job; + if ( + ["cancelled", "completed", "discarded"].includes(job.state) || + job.finalizedAt !== null + ) + return job; + const now = Temporal.Now.instant(); + await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=CASE WHEN state='running' THEN finalized_at ELSE ? END, + metadata=JSON_SET(metadata,'$.cancel_attempted_at',?), state=CASE WHEN state='running' THEN state ELSE 'cancelled' END WHERE id=?`, + [timestamp(now), now.toString(), id] + ); + if (job.state === "running") + await this.#notify( + tx, + "control", + `{"action":"cancel","job_id":${id},"queue":${JSON.stringify(job.queue)}}` + ); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async jobDelete(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return { status: "not_found" }; + const job = decodeJobPartial(row).job; + if (job.state === "running") return { job, status: "running" }; + await this.#write(tx, "DELETE FROM {schema}river_job WHERE id=?", [id]); + return { job, status: "deleted" }; + }); + } + async jobDeleteMany( + params: JobDeleteManyParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const { where, values } = filters(params); + if (!params.all && where.length === 0) + throw new ConfigurationError("Deleting all jobs requires all=true"); + where.push("state != 'running'"); + const rows = await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE ${where.join(" AND ")} ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED`, + [...values, params.limit] + ); + for (const row of rows) + await this.#write(tx, "DELETE FROM {schema}river_job WHERE id=?", [ + row.id, + ]); + return rows.map((row) => decodeJobPartial(row).job); + }); + } + async jobList( + params: JobListParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const { where, values } = filters(params); + if (params.metadata !== null) { + where.push(metadataContains(params.metadata, "metadata", values)); + } + if (params.tagsAll.length) { + where.push("JSON_CONTAINS(tags,CAST(? AS JSON))"); + values.push(json(params.tagsAll)); + } + if (params.tagsAny.length) { + where.push("JSON_OVERLAPS(tags,CAST(? AS JSON))"); + values.push(json(params.tagsAny)); + } + const keyset = jobListKeyset(params); + const order = jobListKeysetSql(keyset, (value) => { + values.push(typeof value === "bigint" ? value : timestamp(value)); + return "?"; + }); + if (order.after) where.push(order.after); + const orderBy = order.orderBy.replace( + /(\w+) (ASC|DESC) NULLS (LAST|FIRST)/g, + (_, field: string, direction: string, nulls: string) => + `${field} IS NULL ${nulls === "LAST" ? "ASC" : "DESC"}, ${field} ${direction}` + ); + return ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE ${where.join(" AND ") || "true"} ORDER BY ${orderBy} LIMIT ?`, + [...values, params.limit] + ) + ).map(decodeJob); + }); + } + async jobRetry(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return null; + const job = decodeJobPartial(row).job; + const now = Temporal.Now.instant(); + if ( + job.state === "running" || + (job.state === "available" && + Temporal.Instant.compare(job.scheduledAt, now) < 0) + ) + return job; + await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=NULL, max_attempts=CASE WHEN attempt=max_attempts THEN max_attempts+1 ELSE max_attempts END, scheduled_at=?, state='available' WHERE id=?`, + [timestamp(now), id] + ); + await this.notifyInsert([job.queue], { tx }); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async #metadata( + tx: PoolConnection, + id: bigint, + metadata: JsonObject + ): Promise { + const entries = Object.entries(metadata); + if (!entries.length) return; + const values = entries.flatMap(([key, value]) => [ + `$.${JSON.stringify(key)}`, + json(value), + ]); + await this.#write( + tx, + `UPDATE {schema}river_job SET metadata=JSON_SET(metadata,${entries.map(() => "?,CAST(? AS JSON)").join(",")}) WHERE id=?`, + [...values, id] + ); + } + async jobUpdate( + id: bigint, + params: JobUpdateParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + if (!(await this.#get(tx, id, true))) return null; + await this.#metadata(tx, id, { + ...params.metadata, + ...(params.output === undefined ? {} : { output: params.output }), + }); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async jobClaim( + params: JobClaimParams, + options: JobClaimOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const jobs: JobRow[] = []; + const decodeErrors = new Map(); + const now = timestamp(Temporal.Now.instant()); + for (const queue of params.queues) { + if (!queue.limit) continue; + const kindSql = params.kinds.length + ? ` AND CAST(kind AS BINARY) IN (${params.kinds.map(() => "?").join(",")})` + : ""; + // Keep an indexable equality so claims do not scan and lock other queues. + const rows = await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? + AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL)${kindSql} + ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED`, + [queue.name, queue.name, now, ...params.kinds, queue.limit] + ); + for (const row of rows) { + await this.#write( + tx, + `UPDATE {schema}river_job SET attempted_at=?, attempted_by=JSON_ARRAY_APPEND(CASE WHEN JSON_TYPE(attempted_by) != 'ARRAY' OR attempted_by IS NULL THEN JSON_ARRAY() WHEN JSON_LENGTH(attempted_by)>=100 THEN JSON_EXTRACT(attempted_by,'$[last-98 to last]') ELSE attempted_by END,'$',?), attempt=attempt+1,state='running' WHERE id=?`, + [now, params.attemptedBy, row.id] + ); + const result = decodeJobPartial( + required(await this.#get(tx, BigInt(String(row.id)), true)) + ); + jobs.push(result.job); + if (result.error) decodeErrors.set(result.job.id, result.error); + } + } + return { jobs, decodeErrors }; + }); + } + async jobCompleteMany( + commands: readonly JobCompletionCommand[], + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const results: JobCompletionResult[] = []; + for (const command of commands) { + const raw = await this.#get(tx, command.id, true); + const job = raw ? decodeJobPartial(raw).job : null; + const key = jobCompletionKey(command); + if ( + !job || + job.attempt !== command.attempt || + job.attemptedBy.at(-1) !== command.attemptedBy + ) { + results.push({ job, key, status: "stale" }); + continue; + } + let state: JobState = + command.kind === "complete" + ? "completed" + : command.kind === "cancel" + ? "cancelled" + : command.kind === "discard" + ? "discarded" + : command.kind === "interrupt" || command.available + ? "available" + : command.kind === "retry" + ? "retryable" + : "scheduled"; + const cancelled = + ["available", "retryable", "scheduled"].includes(state) && + Object.hasOwn(job.metadata, "cancel_attempted_at"); + let finalizedAt = command.finalizedAt; + let scheduledAt = command.scheduledAt ?? job.scheduledAt; + let attempt = + command.kind === "snooze" || command.kind === "interrupt" + ? Math.max(0, command.attempt - 1) + : command.attempt; + if (cancelled) { + state = "cancelled"; + finalizedAt = Temporal.Now.instant(); + scheduledAt = job.scheduledAt; + attempt = job.attempt; + } + if (job.state === "running") { + if ( + ["cancelled", "completed", "discarded"].includes(state) !== + (finalizedAt !== null) + ) + throw new ConfigurationError("Invalid completion timestamp"); + await this.#write( + tx, + `UPDATE {schema}river_job SET attempt=?,finalized_at=?,scheduled_at=?,state=?, + errors=CASE WHEN ? THEN JSON_ARRAY_APPEND(CASE WHEN errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(errors)='ARRAY' THEN errors ELSE JSON_ARRAY(errors) END,'$',CAST(? AS JSON)) ELSE errors END WHERE id=?`, + [ + attempt, + timestamp(finalizedAt), + timestamp(scheduledAt), + state, + command.error !== null, + command.error === null + ? "null" + : json({ + ...command.error, + at: command.error.at.toString(), + attempt: command.attempt, + }), + command.id, + ] + ); + } + await this.#metadata(tx, command.id, { + ...command.metadata, + ...(command.outputSet ? { output: command.output } : {}), + }); + results.push({ + job: decodeJobPartial(required(await this.#get(tx, command.id, true))) + .job, + key, + status: job.state === "running" ? "applied" : "stale", + }); + } + return results; + }); + } + + async #notify( + tx: PoolConnection, + topic: RuntimeNotification["topic"], + payload: string + ): Promise { + await this.#write( + tx, + "INSERT INTO {schema}river_notification(topic,payload) VALUES (?,?)", + [`river_${topic}`, payload] + ); + } + async notifyInsert( + queues: readonly string[], + options: Options = {} + ): Promise { + await this.#transaction(options, async (tx) => { + for (const queue of new Set(queues)) + await this.#notify(tx, "insert", json({ queue })); + }); + } + async jobGetCancelRequested( + ids: readonly bigint[], + options: RuntimeWaitOptions = {} + ): Promise { + if (!ids.length) return []; + return this.#transaction(options, async (tx) => + ( + await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE id IN (${ids.map(() => "?").join(",")}) AND state='running' AND JSON_CONTAINS_PATH(metadata,'one','$.cancel_attempted_at')`, + ids + ) + ).map((row) => BigInt(String(row.id))) + ); + } + async runtimeRequestLeadershipResignation( + options: Options = {} + ): Promise { + await this.#transaction(options, (tx) => + this.#notify( + tx, + "leadership", + json({ action: "request_resign", leader_id: "" }) + ) + ); + } + async *runtimeNotificationSubscribe( + topics: readonly RuntimeNotification["topic"][], + signal: AbortSignal, + ready: () => void + ): AsyncIterable { + try { + let after = await this.#transaction({ signal }, async (tx) => { + const rows = await this.#rows( + tx, + "SELECT CAST(id AS CHAR) AS id FROM {schema}river_notification ORDER BY id FOR UPDATE" + ); + return BigInt(text(rows.at(-1)?.id ?? "0")); + }); + ready(); + while (!signal.aborted) { + const rows = await this.#transaction({ signal }, (tx) => + this.#rows( + tx, + "SELECT CAST(id AS CHAR) AS id,topic,payload FROM {schema}river_notification WHERE id>? ORDER BY id LIMIT 256 FOR SHARE", + [after] + ) + ); + for (const row of rows) { + after = BigInt(String(row.id)); + const topic = String(row.topic).replace( + /^river_/, + "" + ) as RuntimeNotification["topic"]; + if (topics.includes(topic)) + yield { topic, payload: String(row.payload) }; + } + try { + await sleep(25, undefined, { signal }); + } catch (error) { + if (!(error instanceof Error && error.name === "AbortError")) + throw error; + } + } + } catch (error) { + if (!signal.aborted) throw error; + } + } + + async #queue( + tx: PoolConnection, + name: string, + lock = false + ): Promise { + const row = ( + await this.#rows( + tx, + `SELECT ${QUEUE_COLUMNS} FROM {schema}river_queue WHERE name=CAST(? AS BINARY)${lock ? " FOR UPDATE" : ""}`, + [name] + ) + )[0]; + return row ? decodeQueue(row) : null; + } + async queueGet( + name: string, + options: Options = {} + ): Promise { + return this.#transaction(options, (tx) => this.#queue(tx, name)); + } + async queueList( + params: QueueListParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => + ( + await this.#rows( + tx, + `SELECT ${QUEUE_COLUMNS} FROM {schema}river_queue WHERE (? IS NULL OR name>?) ORDER BY name LIMIT ?`, + [params.nameAfter, params.nameAfter, params.limit] + ) + ).map(decodeQueue) + ); + } + async queuePause( + name: string, + options: Options = {} + ): Promise { + return this.#setPaused(name, true, options); + } + async queueResume( + name: string, + options: Options = {} + ): Promise { + return this.#setPaused(name, false, options); + } + async #setPaused( + name: string, + paused: boolean, + options: Options + ): Promise { + return this.#transaction(options, async (tx) => { + const condition = paused ? "paused_at IS NULL" : "paused_at IS NOT NULL"; + await this.#write( + tx, + `UPDATE {schema}river_queue SET updated_at=CASE WHEN ${condition} THEN UTC_TIMESTAMP(6) ELSE updated_at END, paused_at=${paused ? "COALESCE(paused_at,UTC_TIMESTAMP(6))" : "NULL"} WHERE (?='*' OR name=CAST(? AS BINARY))`, + [name, name] + ); + await this.#notify( + tx, + "control", + json({ action: paused ? "pause" : "resume", queue: name }) + ); + return this.#queue(tx, name); + }); + } + async queueUpdate( + name: string, + params: QueueUpdateParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const before = await this.#queue(tx, name, true); + if (!before) return null; + const updated = queueMetadataUpdate(name, params); + if (updated) { + await this.#write( + tx, + "UPDATE {schema}river_queue SET metadata=CAST(? AS JSON),updated_at=UTC_TIMESTAMP(6) WHERE name=CAST(? AS BINARY)", + [updated.text, name] + ); + await this.#notify(tx, "control", updated.notification); + } + return this.#queue(tx, name); + }); + } + async runtimeQueueUpsert( + name: string, + now: Temporal.Instant, + options: RuntimeWaitOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + await this.#write( + tx, + "INSERT INTO {schema}river_queue(name,created_at,updated_at) VALUES (?,?,?) ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at)", + [name, timestamp(now), timestamp(now)] + ); + return required(await this.#queue(tx, name, true)); + }); + } + + async maintenanceLeaderAcquire( + leaderId: string, + now: Temporal.Instant, + ttlMs: number, + held: RuntimeLeader | null, + options: RuntimeWaitOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const expiresAt = now.add({ milliseconds: ttlMs }); + if (held) { + const updated = await this.#write( + tx, + "UPDATE {schema}river_leader SET expires_at=? WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at>?", + [ + timestamp(expiresAt), + held.leaderId, + timestamp(held.electedAt), + timestamp(now), + ] + ); + return updated ? { ...held, expiresAt } : null; + } + await this.#write( + tx, + "DELETE FROM {schema}river_leader WHERE name='default' AND expires_at { + return this.#transaction({}, async (tx) => { + const changed = await this.#write( + tx, + "DELETE FROM {schema}river_leader WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=?", + [leader.leaderId, timestamp(leader.electedAt)] + ); + if (changed) + await this.#notify( + tx, + "leadership", + json({ action: "resigned", leader_id: leader.leaderId }) + ); + return changed > 0; + }); + } + async #fenced( + leader: RuntimeLeader, + options: Options, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#transaction(options, async (tx) => { + const rows = await this.#rows( + tx, + "SELECT leader_id FROM {schema}river_leader WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at>UTC_TIMESTAMP(6) FOR SHARE", + [leader.leaderId, timestamp(leader.electedAt)] + ); + if (!rows.length) return null; + return callback(tx); + }); + } + async maintenanceTransaction( + leader: RuntimeLeader, + batch: RuntimeMaintenanceBatch, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#fenced(leader, { signal: batch.signal }, async (tx) => { + const result = await callback(tx); + batch.signal.throwIfAborted(); + return result; + }); + } + async maintenanceSchedule( + leader: RuntimeLeader, + params: RuntimeScheduleParams, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced( + leader, + batch ? { signal: batch.signal } : {}, + async (tx) => { + const rows = await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE state IN ('retryable','scheduled') AND scheduled_at<=? ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED`, + [timestamp(params.scheduledAtHorizon), params.limit] + ); + const queues: string[] = []; + for (const row of rows) { + const job = decodeJobPartial(row).job; + // A savepoint contains only the conflicting state change, leaving the + // batch usable after MySQL's duplicate-key error. + await tx.query("SAVEPOINT river_schedule"); + try { + await this.#write( + tx, + "UPDATE {schema}river_job SET state='available' WHERE id=?", + [job.id] + ); + if ( + Temporal.Instant.compare( + job.scheduledAt, + params.notificationHorizon + ) <= 0 + ) + queues.push(job.queue); + } catch (error) { + if ((error as { code?: string }).code !== "ER_DUP_ENTRY") + throw error; + await tx.query("ROLLBACK TO SAVEPOINT river_schedule"); + await this.#write( + tx, + "UPDATE {schema}river_job SET state='discarded',finalized_at=?,metadata=JSON_SET(metadata,'$.unique_key_conflict','scheduler_discarded') WHERE id=?", + [timestamp(params.now), job.id] + ); + } finally { + await tx.query("RELEASE SAVEPOINT river_schedule"); + } + } + await this.notifyInsert(params.allowInsertNotifications(queues), { + tx, + }); + return rows.length; + } + )) ?? 0 + ); + } + async maintenanceGetStuck( + leader: RuntimeLeader, + attemptedBefore: Temporal.Instant, + afterId: bigint, + limit: number, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced( + leader, + batch ? { signal: batch.signal } : {}, + async (tx) => + ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE state='running' AND attempted_at? ORDER BY id LIMIT ?`, + [timestamp(attemptedBefore), afterId, limit] + ) + ).map((row) => decodeJobPartial(row).job) + )) ?? [] + ); + } + async maintenanceRescue( + leader: RuntimeLeader, + attemptedBefore: Temporal.Instant, + jobs: readonly RuntimeJobRescue[], + options: Options = {} + ): Promise { + return ( + (await this.#fenced(leader, options, async (tx) => { + let count = 0; + for (const job of jobs) { + const raw = await this.#get(tx, job.id, true); + if (!raw) continue; + const before = decodeJobPartial(raw).job; + if ( + before.state !== "running" || + before.attemptedAt === null || + Temporal.Instant.compare(before.attemptedAt, attemptedBefore) >= 0 + ) + continue; + count += await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=?,scheduled_at=?,state=?,metadata=JSON_SET(metadata,'$."river:rescue_count"',COALESCE(CASE WHEN JSON_TYPE(JSON_EXTRACT(metadata,'$."river:rescue_count"')) IN ('INTEGER','DOUBLE') THEN JSON_EXTRACT(metadata,'$."river:rescue_count"') ELSE NULL END,0)+1),errors=JSON_ARRAY_APPEND(COALESCE(errors,JSON_ARRAY()),'$',CAST(? AS JSON)) WHERE id=?`, + [ + timestamp(job.finalizedAt), + timestamp(job.scheduledAt), + job.state, + json({ ...job.error, at: job.error.at.toString() }), + job.id, + ] + ); + } + return count; + })) ?? 0 + ); + } + async maintenanceCleanJobs( + leader: RuntimeLeader, + params: RuntimeJobCleanupParams, + _timeoutMs: number | null, + signal: AbortSignal + ): Promise { + return ( + (await this.#fenced(leader, { signal }, async (tx) => { + const rows = await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE ((state='cancelled' AND finalized_at "?").join(",")})` : ""} ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED`, + [ + timestamp(params.cancelledBefore), + timestamp(params.completedBefore), + timestamp(params.discardedBefore), + ...(params.queuesExcluded ?? []), + params.limit, + ] + ); + if (!rows.length) return 0; + return this.#write( + tx, + `DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM {schema}river_job WHERE id IN (${rows.map(() => "?").join(",")})`, + rows.map((row) => row.id) + ); + })) ?? 0 + ); + } + async maintenanceCleanQueues( + leader: RuntimeLeader, + updatedBefore: Temporal.Instant, + limit: number, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced(leader, batch ? { signal: batch.signal } : {}, (tx) => + this.#write( + tx, + "DELETE FROM {schema}river_queue WHERE updated_at { + return ( + (await this.#fenced(leader, batch ? { signal: batch.signal } : {}, (tx) => + this.#write( + tx, + "DELETE FROM {schema}river_notification WHERE created_at +): { where: string[]; values: unknown[] } { + const where: string[] = []; + const values: unknown[] = []; + for (const [column, items] of [ + ["id", params.ids], + ["CAST(kind AS BINARY)", params.kinds], + ["priority", params.priorities], + ["CAST(queue AS BINARY)", params.queues], + ["state", params.states], + ] as const) { + if (items.length) { + where.push(`${column} IN (${items.map(() => "?").join(",")})`); + values.push(...items); + } + } + return { where, values }; +} + +function binding(value: unknown): string | number | boolean | Buffer | null { + if (typeof value === "bigint") return value.toString(); + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + Buffer.isBuffer(value) + ) + return value; + throw new TypeError("Invalid MySQL SQL parameter"); +} + +// MySQL JSON_CONTAINS flattens nested arrays and accepts scalar-in-array +// matches. Preserve Postgres containment boundaries by testing each element. +// Scalar subqueries avoid MySQL 8 semijoin rewrites losing nested correlations. +function metadataContains( + wanted: JsonValue, + value: string, + values: unknown[], + depth = 0 +): string { + const alias = `metadata_${depth}`; + if ( + wanted !== null && + typeof wanted === "object" && + !isExactJsonNumber(wanted) + ) { + const array = Array.isArray(wanted); + const children = array + ? wanted.map((item) => { + const predicate = metadataContains( + item, + `${alias}.value`, + values, + depth + 1 + ); + return `(SELECT 1 FROM JSON_TABLE(${value}, '$[*]' COLUMNS(value JSON PATH '$')) AS ${alias} WHERE ${predicate} LIMIT 1) IS NOT NULL`; + }) + : Object.entries(wanted).map(([key, item]) => { + values.push(`$.${JSON.stringify(key)}`); + const predicate = metadataContains( + item, + `${alias}.value`, + values, + depth + 1 + ); + return `(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT(${value},?), '$' COLUMNS(value JSON PATH '$')) AS ${alias} WHERE ${predicate} LIMIT 1) IS NOT NULL`; + }); + return `(JSON_TYPE(${value})='${array ? "ARRAY" : "OBJECT"}' AND ${children.join(" AND ") || "true"})`; + } + if (wanted === null) + return `(${value} IS NULL OR JSON_TYPE(${value})='NULL')`; + values.push(json(wanted)); + return `(JSON_TYPE(${value}) NOT IN ('ARRAY','OBJECT') AND JSON_CONTAINS(${value},CAST(? AS JSON)))`; +} + +function required(value: T | null | undefined): T { + if (value == null) throw new Error("MySQL row disappeared while locked"); + return value; +} diff --git a/js/driver/mysql/src/index.ts b/js/driver/mysql/src/index.ts new file mode 100644 index 000000000..4c11778b0 --- /dev/null +++ b/js/driver/mysql/src/index.ts @@ -0,0 +1,11 @@ +import type { PoolConnection } from "mysql2/promise"; + +export { MySqlDriver } from "./driver.js"; +export type { MySqlDriverOptions } from "./driver.js"; + +declare module "riverqueue" { + interface RiverTransactionRegistry { + /** A mysql2 connection with an open transaction. */ + "@riverqueue/driver-mysql": PoolConnection; + } +} diff --git a/js/driver/mysql/tsconfig.json b/js/driver/mysql/tsconfig.json new file mode 100644 index 000000000..fc6b18f33 --- /dev/null +++ b/js/driver/mysql/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "stripInternal": true + }, + "exclude": ["src/**/*.integration.test.ts", "src/**/*.test.ts"], + "include": ["src"] +} diff --git a/js/etc/riverqueue.api.md b/js/etc/riverqueue.api.md index 8418e3a83..4fa247a07 100644 --- a/js/etc/riverqueue.api.md +++ b/js/etc/riverqueue.api.md @@ -2638,6 +2638,11 @@ export interface RiverPlugin { */ export interface RiverTransactionRegistry {} +interface RiverTransactionRegistry { + /** A mysql2 connection with an open transaction. */ + "@riverqueue/driver-mysql": PoolConnection; +} + interface RiverTransactionRegistry { /** node-postgres clients (`pg.Client` or a pool's `PoolClient`). */ "@riverqueue/driver-pg": ClientBase; diff --git a/js/etc/riverqueue.unstable-driver.api.md b/js/etc/riverqueue.unstable-driver.api.md index 272ff2394..293a5e4dd 100644 --- a/js/etc/riverqueue.unstable-driver.api.md +++ b/js/etc/riverqueue.unstable-driver.api.md @@ -212,6 +212,11 @@ export declare function driverMigrationTarget( * a `node:sqlite` database. */ export type DriverMigrationTarget = + | { + readonly backend: "mysql"; + readonly pool: object; + readonly schema: string | undefined; + } | { readonly client: object; readonly schema: string | undefined; diff --git a/js/migrate/etc/migrate.api.md b/js/migrate/etc/migrate.api.md index cc56ac21e..279e8d97a 100644 --- a/js/migrate/etc/migrate.api.md +++ b/js/migrate/etc/migrate.api.md @@ -135,7 +135,7 @@ export declare const MIGRATION_LINE_MAIN = "main"; ```ts /** Database backend that a migration bundle targets. */ -export type MigrationBackend = "postgres" | "sqlite"; +export type MigrationBackend = "mysql" | "postgres" | "sqlite"; ``` ## `MigrationDirection` @@ -162,7 +162,10 @@ export declare class MigrationError extends RiverError<"migration"> { ```ts /** A database connection to migrate, given without a River driver. */ export type MigrationTarget = - PgClientMigrationTarget | PgPoolMigrationTarget | SqliteMigrationTarget; + | MySqlMigrationTarget + | PgClientMigrationTarget + | PgPoolMigrationTarget + | SqliteMigrationTarget; ``` ## `Migrator` @@ -239,6 +242,36 @@ export interface MigratorOptions { export type MigratorSource = ClientDriver | MigrationTarget; ``` +## `MySqlMigrationConnection` + +```ts +/** The mysql2 promise connection methods used by the migrator. */ +export interface MySqlMigrationConnection { + query(sql: string, values?: unknown[]): Promise<[unknown, unknown]>; + release(): void; +} +``` + +## `MySqlMigrationPool` + +```ts +/** A caller-owned mysql2 promise pool. */ +export interface MySqlMigrationPool { + getConnection(): Promise; +} +``` + +## `MySqlMigrationTarget` + +```ts +/** Migrate MySQL through a caller-owned mysql2 pool. */ +export interface MySqlMigrationTarget { + readonly backend: "mysql"; + readonly pool: MySqlMigrationPool; + readonly schema?: string | undefined; +} +``` + ## `PgClientMigrationTarget` ```ts diff --git a/js/migrate/migrations/manifest.json b/js/migrate/migrations/manifest.json index ef9f6f0af..61b72abed 100644 --- a/js/migrate/migrations/manifest.json +++ b/js/migrate/migrations/manifest.json @@ -1,5 +1,71 @@ { "backends": { + "mysql": [ + { + "file": "001_create_river_migration.down.sql", + "sha256": "660d6b11eeb88589d3ce2053be165cda8df4c1d0172fbc1e8c3daa614e3f5b93" + }, + { + "file": "001_create_river_migration.up.sql", + "sha256": "e8821d3f7e89892d726fd94b278316e82835995cc6bf5def4641137de930f68c" + }, + { + "file": "002_initial_schema.down.sql", + "sha256": "93d74166cb838b13ae42447181afb2c2987dcb16ae33422de612daecb0163e79" + }, + { + "file": "002_initial_schema.up.sql", + "sha256": "dbe917102df1708a316b89c789a587d358fad11d6c8dac26f95a22e8691ad359" + }, + { + "file": "003_river_job_tags_non_null.down.sql", + "sha256": "dbfb0192d6254bdcf2a12c70c8b4ad44dfd7ebaa273b3160c0e76b9208f0a1fe" + }, + { + "file": "003_river_job_tags_non_null.up.sql", + "sha256": "e8a8459edd8e6110e564be74c03f646b430e2f3e31fef31eeb5610ed3b10660f" + }, + { + "file": "004_pending_and_more.down.sql", + "sha256": "f4325dd580c3725d7eb068262d56319f779b556ceddfb53771c51a855ca2440f" + }, + { + "file": "004_pending_and_more.up.sql", + "sha256": "21711fa26eeb7c3abb9cb440652ba4a79fac52bb6ae1e1e5204b59e8b5847718" + }, + { + "file": "005_migration_unique_client.down.sql", + "sha256": "2ba63bb2563f28f4a1fe1713c3336feb46dacf979079d0859682ec1e83b72cc5" + }, + { + "file": "005_migration_unique_client.up.sql", + "sha256": "a7634888e5e8f25121d8c75592451106bf7a2d7ab15c037b561d7650908b0688" + }, + { + "file": "006_bulk_unique.down.sql", + "sha256": "5c86b3bd5e32a1f8462b6e4f7ba8aa4fcb4c5be37f4c7d3c1faeb9602ca410fe" + }, + { + "file": "006_bulk_unique.up.sql", + "sha256": "f8625bc2fbff65b1b77fb416bc95889981256d890ce8d157bb1227064bff5228" + }, + { + "file": "007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql", + "sha256": "897217327df7fc1d85b57a3c785d70b4a33eae9c3957f7977b0a0365de826a17" + }, + { + "file": "007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql", + "sha256": "546a783aec00f2b2fdac03d2a902f4899c223d5789602091a31be166c3eebf18" + }, + { + "file": "008_job_id_autoincrement.down.sql", + "sha256": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa" + }, + { + "file": "008_job_id_autoincrement.up.sql", + "sha256": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa" + } + ], "postgres": [ { "file": "001_create_river_migration.down.sql", @@ -135,6 +201,7 @@ }, "format": 1, "sources": { + "mysql": "riverdriver/rivermysql/migration/main", "postgres": "riverdriver/riverpgxv5/migration/main", "sqlite": "riverdriver/riversqlite/migration/main" } diff --git a/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql b/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql b/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..9aee5ff2f --- /dev/null +++ b/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/js/migrate/migrations/mysql/main/002_initial_schema.down.sql b/js/migrate/migrations/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/js/migrate/migrations/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/js/migrate/migrations/mysql/main/002_initial_schema.up.sql b/js/migrate/migrations/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..c680cb3db --- /dev/null +++ b/js/migrate/migrations/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify is +-- omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB; diff --git a/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql b/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql b/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..7725c9f12 --- /dev/null +++ b/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. MySQL doesn't have enum types to +-- alter, so we update the CHECK constraint instead. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql b/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..108c807a8 --- /dev/null +++ b/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql b/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..fc5889c45 --- /dev/null +++ b/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; diff --git a/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql b/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql b/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..c4c740ea7 --- /dev/null +++ b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..d5dc9a6b4 --- /dev/null +++ b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/js/migrate/src/bundle.ts b/js/migrate/src/bundle.ts index 11c2f1e43..07234fbe7 100644 --- a/js/migrate/src/bundle.ts +++ b/js/migrate/src/bundle.ts @@ -4,7 +4,7 @@ import { readFileSync } from "node:fs"; import { MigrationError } from "riverqueue"; /** Database backend that a migration bundle targets. */ -export type MigrationBackend = "postgres" | "sqlite"; +export type MigrationBackend = "mysql" | "postgres" | "sqlite"; /** One River migration version with SQL for both directions. */ export interface Migration { @@ -42,7 +42,7 @@ const MIGRATION_FILE_RE = export function loadMigrations( backend: MigrationBackend ): readonly Migration[] { - if (!["postgres", "sqlite"].includes(backend)) { + if (!["mysql", "postgres", "sqlite"].includes(backend)) { throw new MigrationError( `unsupported migration backend: ${JSON.stringify(backend)}`, { backend, operation: "load" } diff --git a/js/migrate/src/index.ts b/js/migrate/src/index.ts index a27bd2e70..820e3efb9 100644 --- a/js/migrate/src/index.ts +++ b/js/migrate/src/index.ts @@ -18,6 +18,7 @@ export type { Migrator, MigratorOptions, MigratorSource, + MySqlMigrationTarget, PgClientMigrationTarget, PgPoolMigrationTarget, SqliteMigrationTarget, @@ -30,3 +31,5 @@ export type { PgMigrationPoolClient, PgMigrationQueryResult, } from "./postgres.js"; + +export type { MySqlMigrationConnection, MySqlMigrationPool } from "./mysql.js"; diff --git a/js/migrate/src/migrator.ts b/js/migrate/src/migrator.ts index a15397064..6d6ae9e52 100644 --- a/js/migrate/src/migrator.ts +++ b/js/migrate/src/migrator.ts @@ -21,6 +21,7 @@ import { SqliteMigrationStorage, type SqliteMigrationRunner, } from "./sqlite.js"; +import { MySqlMigrationStorage, type MySqlMigrationPool } from "./mysql.js"; import type { MigrationStorage } from "./storage.js"; export { MIGRATION_LINE_MAIN }; @@ -137,7 +138,17 @@ export type MigratorSource = ClientDriver | MigrationTarget; /** A database connection to migrate, given without a River driver. */ export type MigrationTarget = - PgClientMigrationTarget | PgPoolMigrationTarget | SqliteMigrationTarget; + | MySqlMigrationTarget + | PgClientMigrationTarget + | PgPoolMigrationTarget + | SqliteMigrationTarget; + +/** Migrate MySQL through a caller-owned mysql2 pool. */ +export interface MySqlMigrationTarget { + readonly backend: "mysql"; + readonly pool: MySqlMigrationPool; + readonly schema?: string | undefined; +} /** Migrate Postgres through one dedicated connection. */ export interface PgClientMigrationTarget { @@ -216,12 +227,16 @@ export function createMigrator( ): Migrator { const target = resolveTarget(source); const storage: MigrationStorage = - "database" in target - ? new SqliteMigrationStorage(target.database, target.run) - : new PgMigrationStorage( - "pool" in target ? { pool: target.pool } : { client: target.client }, - target.schema - ); + "backend" in target + ? new MySqlMigrationStorage(target.pool, target.schema) + : "database" in target + ? new SqliteMigrationStorage(target.database, target.run) + : new PgMigrationStorage( + "pool" in target + ? { pool: target.pool } + : { client: target.client }, + target.schema + ); const line = options.line ?? MIGRATION_LINE_MAIN; validateLine(storage.backend, line); const migrations = @@ -420,6 +435,7 @@ function copyMigrations( function resolveTarget( source: MigratorSource ): + | MySqlMigrationTarget | PgClientMigrationTarget | PgPoolMigrationTarget | (SqliteMigrationTarget & { readonly run?: SqliteMigrationRunner }) { @@ -437,6 +453,7 @@ function resolveTarget( }; } if (typeof source === "object" && (source as unknown) !== null) { + if ("backend" in source) return source; if ("database" in source) return { database: source.database }; if ("pool" in source) { return { pool: source.pool, schema: source.schema }; diff --git a/js/migrate/src/mysql.ts b/js/migrate/src/mysql.ts new file mode 100644 index 000000000..e49efac68 --- /dev/null +++ b/js/migrate/src/mysql.ts @@ -0,0 +1,199 @@ +import { MigrationError } from "riverqueue"; +import { isInTargetState, requireLineStorage, versionRecord } from "./plan.js"; +import type { + MigrationSession, + MigrationStep, + MigrationStorage, +} from "./storage.js"; + +/** The mysql2 promise connection methods used by the migrator. */ +export interface MySqlMigrationConnection { + query(sql: string, values?: unknown[]): Promise<[unknown, unknown]>; + release(): void; +} +/** A caller-owned mysql2 promise pool. */ +export interface MySqlMigrationPool { + getConnection(): Promise; +} + +/** MySQL DDL commits implicitly; the named lock spans all migration steps. */ +export class MySqlMigrationStorage implements MigrationStorage { + readonly backend = "mysql"; + readonly #pool: MySqlMigrationPool; + readonly #schema: string | undefined; + readonly #prefix: string; + constructor(pool: MySqlMigrationPool, schema: string | undefined) { + if (schema !== undefined && !/^[A-Za-z_][A-Za-z0-9_]{0,45}$/.test(schema)) + throw new MigrationError("Invalid MySQL schema name", { + backend: "mysql", + operation: "configure", + }); + this.#pool = pool; + this.#schema = schema; + this.#prefix = schema === undefined ? "" : `\`${schema}\`.`; + } + renderSql(sql: string): string { + return sql.replaceAll("/* TEMPLATE: schema */", this.#prefix); + } + async readVersions(line: string): Promise { + const connection = await this.#pool.getConnection(); + try { + return await this.#versions(connection, line); + } finally { + connection.release(); + } + } + async #versions( + connection: MySqlMigrationConnection, + line: string + ): Promise { + const rows = await query( + connection, + "SELECT column_name FROM information_schema.columns WHERE table_schema=COALESCE(?,DATABASE()) AND table_name='river_migration'", + [this.#schema ?? null] + ); + const tableExists = rows.length > 0; + const hasLineColumn = rows.some((row) => row.column_name === "line"); + requireLineStorage("mysql", line, { tableExists, hasLineColumn }); + if (!tableExists) return []; + return ( + await query( + connection, + `SELECT version FROM ${this.#prefix}river_migration${hasLineColumn ? " WHERE line=?" : ""} ORDER BY version`, + hasLineColumn ? [line] : [] + ) + ).map((row) => Number(row.version)); + } + async withSession( + run: (session: MigrationSession) => Promise + ): Promise { + const connection = await this.#pool.getConnection(); + let locked = false; + try { + const result = await query( + connection, + "SELECT GET_LOCK(CONCAT('river_migrate:',COALESCE(?,DATABASE())),0) AS acquired", + [this.#schema ?? null] + ); + if (Number(result[0]?.acquired) !== 1) + throw new Error("Another migrator holds the database lock"); + locked = true; + return await run({ + readVersions: (line) => this.#versions(connection, line), + apply: (step) => this.#apply(connection, step), + }); + } catch (cause) { + if (cause instanceof MigrationError) throw cause; + throw new MigrationError("MySQL migration failed", { + backend: "mysql", + operation: "migrate", + cause, + }); + } finally { + try { + if (locked) + await connection.query( + "SELECT RELEASE_LOCK(CONCAT('river_migrate:',COALESCE(?,DATABASE())))", + [this.#schema ?? null] + ); + } finally { + connection.release(); + } + } + } + async #apply( + connection: MySqlMigrationConnection, + step: MigrationStep + ): Promise { + const { direction, line, migration } = step; + if ( + isInTargetState( + direction, + await this.#versions(connection, line), + migration.version + ) + ) + return false; + if (line === "main" && direction === "down" && migration.version <= 5) { + const rows = await query( + connection, + "SELECT column_name FROM information_schema.columns WHERE table_schema=COALESCE(?,DATABASE()) AND table_name='river_migration' AND column_name='line'", + [this.#schema ?? null] + ); + if ( + rows.length && + ( + await query( + connection, + `SELECT 1 FROM ${this.#prefix}river_migration WHERE line<>'main' LIMIT 1` + ) + ).length + ) + throw new Error("Migrate other lines down before the main line"); + } + if (direction === "up" && this.#schema !== undefined) + await connection.query( + `CREATE DATABASE IF NOT EXISTS \`${this.#schema}\`` + ); + for (const sql of statements(step.sql)) await connection.query(sql); + // Record only successful DDL. MySQL cannot roll back partially applied DDL. + const record = versionRecord(direction, line, migration.version); + switch (record.kind) { + case "none": + break; + case "insert": + await connection.query( + `INSERT INTO ${this.#prefix}river_migration(line,version) VALUES (?,?)`, + [record.line, record.version] + ); + break; + case "insert_without_line": + await connection.query( + `INSERT INTO ${this.#prefix}river_migration(version) VALUES (?)`, + [record.version] + ); + break; + case "delete": + await connection.query( + `DELETE FROM ${this.#prefix}river_migration WHERE line=? AND version=?`, + [record.line, record.version] + ); + break; + case "delete_without_line": + await connection.query( + `DELETE FROM ${this.#prefix}river_migration WHERE version=?`, + [record.version] + ); + break; + } + return true; + } +} + +async function query( + connection: MySqlMigrationConnection, + sql: string, + values: unknown[] = [] +): Promise[]> { + const [rows] = await connection.query(sql, values); + return rows as Record[]; +} + +/** Split scripts without requiring MULTI_STATEMENTS on application pools. */ +function statements(sql: string): string[] { + const tokens = + sql.match( + /'(?:''|\\.|[^'\\])*'|"(?:""|\\.|[^"\\])*"|`(?:``|[^`])*`|--[^\n]*|#[^\n]*|\/\*[\s\S]*?\*\/|;|[\s\S]/g + ) ?? []; + const result: string[] = []; + let buffer = ""; + for (const token of tokens) { + if (/^(--|#|\/\*)/.test(token)) buffer += " "; + else if (token === ";") { + if (buffer.trim()) result.push(buffer.trim()); + buffer = ""; + } else buffer += token; + } + if (buffer.trim()) result.push(buffer.trim()); + return result; +} diff --git a/js/package.json b/js/package.json index a617dfbff..846eff95c 100644 --- a/js/package.json +++ b/js/package.json @@ -101,6 +101,7 @@ "fast-check": "^4.10.2", "knip": "^6.38.0", "license-checker-rseidelsohn": "^5.0.1", + "mysql2": "^3.24.4", "pg": "^8.22.0", "pino": "^10.3.1", "prettier": "^3.9.6", diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 742e4fed1..dbc764e92 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: license-checker-rseidelsohn: specifier: ^5.0.1 version: 5.0.1 + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.1.1) pg: specifier: ^8.22.0 version: 8.22.0 @@ -94,6 +97,9 @@ importers: '@riverqueue/migrate': specifier: workspace:0.2.0 version: link:../migrate + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.1.1) pg: specifier: ^8.22.0 version: 8.22.0 @@ -111,6 +117,18 @@ importers: specifier: ^6.0.3 version: 6.0.3 + driver/mysql: + devDependencies: + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.1.1) + riverqueue: + specifier: workspace:0.2.0 + version: link:../.. + typescript: + specifier: ^6.0.3 + version: 6.0.3 + driver/pg: dependencies: postgres-array: diff --git a/js/scripts/api-reports.mjs b/js/scripts/api-reports.mjs index 843abbd8c..818e9344a 100644 --- a/js/scripts/api-reports.mjs +++ b/js/scripts/api-reports.mjs @@ -76,6 +76,11 @@ const reports = [ name: "@riverqueue/cli", output: "cli/etc/cli.api.md", }, + { + entry: "driver/mysql/dist/index.d.ts", + name: "@riverqueue/driver-mysql", + output: "driver/mysql/etc/driver-mysql.api.md", + }, { entry: "driver/pg/dist/index.d.ts", name: "@riverqueue/driver-pg", diff --git a/js/scripts/check-licenses.mjs b/js/scripts/check-licenses.mjs index 62e7645cc..5933670cb 100644 --- a/js/scripts/check-licenses.mjs +++ b/js/scripts/check-licenses.mjs @@ -9,6 +9,7 @@ const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const packageDirectories = [ ".", "migrate", + "driver/mysql", "driver/pg", "driver/prisma", "driver/sqlite", diff --git a/js/scripts/check-packages.mjs b/js/scripts/check-packages.mjs index d97cbf098..233d73a71 100644 --- a/js/scripts/check-packages.mjs +++ b/js/scripts/check-packages.mjs @@ -37,6 +37,7 @@ const rootPackageJson = JSON.parse( const packageSpecs = [ { directory: ".", name: "riverqueue" }, { directory: "migrate", name: "@riverqueue/migrate" }, + { directory: "driver/mysql", name: "@riverqueue/driver-mysql" }, { directory: "driver/pg", name: "@riverqueue/driver-pg" }, { directory: "driver/prisma", name: "@riverqueue/driver-prisma" }, { directory: "driver/sqlite", name: "@riverqueue/driver-sqlite" }, @@ -378,8 +379,10 @@ async function verifyConsumers(archiveByPackageName) { )}\n` ); // Install without type packages first: JavaScript consumers must not need - // them, and the published metadata must not pull them in. - await npmInstall(consumerDirectory); + // them. mysql2 declares @types/node as a required peer, so omit automatic + // peer installation here to exercise consumers without type packages. + // inspectDependencies checks that River itself keeps typings optional. + await npmInstall(consumerDirectory, ["--omit=peer"]); for (const typesPackage of ["@types/node", "@types/pg"]) { await assert.rejects( stat(join(consumerDirectory, "node_modules", typesPackage)), @@ -393,6 +396,7 @@ async function verifyConsumers(archiveByPackageName) { ` import * as river from "riverqueue"; import * as cli from "@riverqueue/cli"; +import * as mysql from "@riverqueue/driver-mysql"; import * as pg from "@riverqueue/driver-pg"; import * as prisma from "@riverqueue/driver-prisma"; import * as sqlite from "@riverqueue/driver-sqlite"; @@ -400,7 +404,7 @@ import * as migrate from "@riverqueue/migrate"; import * as testing from "@riverqueue/test"; import * as workerThreads from "@riverqueue/worker-threads"; -for (const packageNamespace of [river, cli, pg, prisma, sqlite, migrate, testing, workerThreads]) { +for (const packageNamespace of [river, cli, mysql, pg, prisma, sqlite, migrate, testing, workerThreads]) { if (Object.keys(packageNamespace).length === 0) throw new Error("empty package namespace"); } @@ -449,6 +453,7 @@ if (cron("@every 1s").next(start).toString() !== "2026-10-11T12:34:57Z") { const packageNames = [ "riverqueue", "@riverqueue/cli", + "@riverqueue/driver-mysql", "@riverqueue/driver-pg", "@riverqueue/driver-prisma", "@riverqueue/driver-sqlite", diff --git a/js/scripts/check-readme-snippets.mjs b/js/scripts/check-readme-snippets.mjs index a7b94a40a..712508a97 100644 --- a/js/scripts/check-readme-snippets.mjs +++ b/js/scripts/check-readme-snippets.mjs @@ -13,6 +13,7 @@ const readmes = [ .sort() .map((name) => `docs/${name}`), "cli/README.md", + "driver/mysql/README.md", "driver/pg/README.md", "driver/prisma/README.md", "driver/sqlite/README.md", @@ -93,6 +94,9 @@ function checkProgram(directory, virtualFiles, readme) { outDir: undefined, paths: { "@riverqueue/cli": [resolve(repositoryRoot, "cli/dist/index.d.ts")], + "@riverqueue/driver-mysql": [ + resolve(repositoryRoot, "driver/mysql/dist/index.d.ts"), + ], "@riverqueue/driver-pg": [ resolve(repositoryRoot, "driver/pg/dist/index.d.ts"), ], diff --git a/js/scripts/sync-migrations.mjs b/js/scripts/sync-migrations.mjs index 2373b05f1..9bed1cb2b 100644 --- a/js/scripts/sync-migrations.mjs +++ b/js/scripts/sync-migrations.mjs @@ -14,6 +14,7 @@ const riverRoot = resolve(repositoryRoot, ".."); const targetRoot = resolve(repositoryRoot, "migrate/migrations"); const check = process.argv.includes("--check"); const sources = { + mysql: "riverdriver/rivermysql/migration/main", postgres: "riverdriver/riverpgxv5/migration/main", sqlite: "riverdriver/riversqlite/migration/main", }; diff --git a/js/src/pilot.ts b/js/src/pilot.ts index 7aa882cf6..8d4a1431e 100644 --- a/js/src/pilot.ts +++ b/js/src/pilot.ts @@ -738,6 +738,11 @@ export type PilotFactory< * a `node:sqlite` database. */ export type DriverMigrationTarget = + | { + readonly backend: "mysql"; + readonly pool: object; + readonly schema: string | undefined; + } | { readonly client: object; readonly schema: string | undefined } | { readonly database: object; diff --git a/js/tsconfig.tests.json b/js/tsconfig.tests.json index 7c29bc586..45f013ac7 100644 --- a/js/tsconfig.tests.json +++ b/js/tsconfig.tests.json @@ -20,6 +20,9 @@ ], "@riverqueue/worker-threads": [ "./worker-threads/src/index.ts" + ], + "@riverqueue/driver-mysql": [ + "./driver/mysql/src/index.ts" ] } }, diff --git a/js/typedoc.json b/js/typedoc.json index 4a11f0bc1..1e2c478c0 100644 --- a/js/typedoc.json +++ b/js/typedoc.json @@ -3,6 +3,7 @@ "entryPoints": [ "src/index.ts", "cli/src/index.ts", + "driver/mysql/src/index.ts", "driver/pg/src/index.ts", "driver/prisma/src/index.ts", "driver/sqlite/src/index.ts", diff --git a/riverdbtest/riverdbtest.go b/riverdbtest/riverdbtest.go index 64468ca9a..652e4d549 100644 --- a/riverdbtest/riverdbtest.go +++ b/riverdbtest/riverdbtest.go @@ -481,7 +481,8 @@ type TestTxOpts struct { // run using TestSchema. This is meant for environments where parallelism // doesn't work as well, like SQLite, which will emit "busy" errors when // multiple clients try to share a schema, even when they're in separate - // transactions. + // transactions. Also applies to MySQL, where InnoDB deadlocks are common + // when multiple transactions are sharing a database. DisableSchemaSharing bool // IsTestTxHelper should be set to true for if TestTx is being called from diff --git a/riverdriver/river_driver_interface.go b/riverdriver/river_driver_interface.go index f76adc29b..231f3e660 100644 --- a/riverdriver/river_driver_interface.go +++ b/riverdriver/river_driver_interface.go @@ -25,6 +25,7 @@ import ( const AllQueuesString = "*" const ( + DatabaseNameMySQL = "mysql" DatabaseNamePostgres = "postgres" DatabaseNameSQLite = "sqlite" ) @@ -53,7 +54,7 @@ var ( type Driver[TTx any] interface { // ArgPlaceholder is the placeholder character used in query positional // arguments, so "$" for "$1", "$2", "$3", etc. This is a "$" for Postgres - // and "?" for SQLite. + // and "?" for MySQL and SQLite. // // API is not stable. DO NOT USE. ArgPlaceholder() string @@ -128,6 +129,13 @@ type Driver[TTx any] interface { // API is not stable. DO NOT USE. PoolSet(dbPool any) error + // SafeIdentifier returns a safely quoted identifier (e.g. a table or + // schema name) for use in SQL queries. Each driver quotes using its + // native syntax: double quotes for Postgres/SQLite, backticks for MySQL. + // + // API is not stable. DO NOT USE. + SafeIdentifier(ident string) string + // SQLFragmentColumnContainsAll generates an SQL fragment to be included as // a predicate in a `WHERE` query for a collection column containing all of // the given values. Postgres uses array containment while SQLite compares @@ -144,6 +152,12 @@ type Driver[TTx any] interface { // API is not stable. DO NOT USE. SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) + // SQLFragmentColumnContainsJSON generates a structural JSON containment + // predicate and its named arguments. Column and namedArg are trusted SQL identifiers. + // + // API is not stable. DO NOT USE. + SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) + // SQLFragmentColumnIn generates an SQL fragment to be included as a // predicate in a `WHERE` query for the existence of a set of values in a // column like `id IN (...)`. The actual implementation depends on support diff --git a/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go b/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go index 4bd6cbfab..887d24f53 100644 --- a/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go +++ b/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go @@ -1521,7 +1521,9 @@ WITH job_input AS ( unnest($6::boolean[]) AS finalized_at_do_update, unnest($7::timestamptz[]) AS finalized_at, unnest($8::boolean[]) AS metadata_do_merge, - unnest($9::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest($9::text[])::jsonb AS metadata_updates, unnest($10::boolean[]) AS scheduled_at_do_update, unnest($11::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] diff --git a/riverdriver/riverdatabasesql/river_database_sql_driver.go b/riverdriver/riverdatabasesql/river_database_sql_driver.go index 0671e852f..be0b9b316 100644 --- a/riverdriver/riverdatabasesql/river_database_sql_driver.go +++ b/riverdriver/riverdatabasesql/river_database_sql_driver.go @@ -90,8 +90,9 @@ func NewWithPgxListener(dbPool *sql.DB, listenerPool *pgxpool.Pool) *Driver { const argPlaceholder = "$" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d} @@ -132,6 +133,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] return fmt.Sprintf("%s && @%s", column, namedArg), pq.Array(values), nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return column + " @> @" + namedArg + "::jsonb", map[string]any{namedArg: fragment}, nil +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { // Identical to the Pgx implementation except for use of `pg.Array`. return fmt.Sprintf("%s = any(@%s)", column, column), pq.Array(values), nil diff --git a/riverdriver/riverdrivertest/client_cancel_test.go b/riverdriver/riverdrivertest/client_cancel_test.go index 601c5c882..bb01449ba 100644 --- a/riverdriver/riverdrivertest/client_cancel_test.go +++ b/riverdriver/riverdrivertest/client_cancel_test.go @@ -24,6 +24,9 @@ func exerciseClientCancelRunningJob[TTx any](ctx context.Context, t *testing.T, config := newTestConfig(t, schema) config.FetchPollInterval = time.Minute + // Startup maintenance may briefly lock a job and make the first SKIP LOCKED + // fetch miss it. Keep this test focused on cancellation after that fetch. + config.LeaderElectionDisabled = true config.PollOnly = pollOnly config.Queues = map[string]river.QueueConfig{river.QueueDefault: {MaxWorkers: 1}} diff --git a/riverdriver/riverdrivertest/client_job_update_test.go b/riverdriver/riverdrivertest/client_job_update_test.go index e1b185e33..927565ce4 100644 --- a/riverdriver/riverdrivertest/client_job_update_test.go +++ b/riverdriver/riverdrivertest/client_job_update_test.go @@ -11,6 +11,7 @@ import ( "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/testfactory" "github.com/riverqueue/river/rivertype" ) @@ -155,4 +156,106 @@ func exerciseClientJobUpdate[TTx any](ctx context.Context, t *testing.T, require.Nil(t, loser.row.FinalizedAt, "the loser of the race condition must observe the winner's finalization clear, not its own snapshot's") }) + t.Run("JobScheduleSkipsLockedJobs", func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t) + if driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL skips locked scheduler candidates; other drivers may wait for them") + } + exec := driver.GetExecutor() + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + ScheduledAt: new(time.Now().Add(-time.Minute)), Schema: schema, State: new(rivertype.JobStateScheduled), + }) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ID: job.ID, Schema: schema}) + require.NoError(t, err) + + scheduleCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + rows, err := exec.JobSchedule(scheduleCtx, &riverdriver.JobScheduleParams{Max: 10, Schema: schema}) + require.NoError(t, err) + require.Empty(t, rows) + require.NoError(t, tx.Rollback(ctx)) + + rows, err = exec.JobSchedule(ctx, &riverdriver.JobScheduleParams{Max: 10, Schema: schema}) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, job.ID, rows[0].Job.ID) + }) + + // A MySQL transaction can already have a REPEATABLE READ snapshot when it + // invokes a state change. No-op updates must still return the current row. + t.Run("JobUpdateWithExistingSnapshot", func(t *testing.T) { + t.Parallel() + + for _, operation := range []string{"Cancel", "DeleteMany", "Retry", "SetStateIfRunning"} { + t.Run(operation, func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t) + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite does not permit a second writer with an open read transaction") + } + exec := driver.GetExecutor() + state := rivertype.JobStateScheduled + switch operation { + case "Retry": + state = rivertype.JobStateCancelled + case "SetStateIfRunning": + state = rivertype.JobStateRunning + } + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Schema: schema, State: &state}) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: job.ID, Schema: schema}) + require.NoError(t, err) + + var current *rivertype.JobRow + switch operation { + case "Cancel": + params := &riverdriver.JobCancelParams{ID: job.ID, CancelAttemptedAt: time.Now(), Schema: schema} + _, err = exec.JobCancel(ctx, params) + require.NoError(t, err) + current, err = tx.JobCancel(ctx, params) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateCancelled, current.State) + case "DeleteMany": + _, err = exec.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ + ID: job.ID, Schema: schema, State: rivertype.JobStateRunning, StateDoUpdate: true, + }) + require.NoError(t, err) + rows, err := tx.JobDeleteMany(ctx, &riverdriver.JobDeleteManyParams{ + Max: 10, OrderByClause: "id", Schema: schema, WhereClause: "true", + }) + require.NoError(t, err) + require.Empty(t, rows, "jobs that are now running must not be deleted") + case "Retry": + params := &riverdriver.JobRetryParams{ID: job.ID, Schema: schema} + _, err = exec.JobRetry(ctx, params) + require.NoError(t, err) + current, err = tx.JobRetry(ctx, params) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateAvailable, current.State) + case "SetStateIfRunning": + _, err = exec.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ + ID: job.ID, FinalizedAt: new(time.Now()), FinalizedAtDoUpdate: true, Schema: schema, + State: rivertype.JobStateCompleted, StateDoUpdate: true, + }) + require.NoError(t, err) + rows, err := tx.JobSetStateIfRunningMany(ctx, &riverdriver.JobSetStateIfRunningManyParams{ + ID: []int64{job.ID}, Attempt: []*int{nil}, ErrData: [][]byte{nil}, FinalizedAt: []*time.Time{new(time.Now())}, + MetadataDoMerge: []bool{false}, MetadataUpdates: [][]byte{nil}, ScheduledAt: []*time.Time{nil}, + Schema: schema, State: []rivertype.JobState{rivertype.JobStateCancelled}, + }) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, rivertype.JobStateCompleted, rows[0].State) + } + }) + } + }) } diff --git a/riverdriver/riverdrivertest/driver_client_test.go b/riverdriver/riverdrivertest/driver_client_test.go index 5dca04ec4..8f16d2a11 100644 --- a/riverdriver/riverdrivertest/driver_client_test.go +++ b/riverdriver/riverdrivertest/driver_client_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + _ "github.com/go-sql-driver/mysql" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/stdlib" "github.com/lib/pq" @@ -22,6 +23,7 @@ import ( "github.com/riverqueue/river/riverdbtest" "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/riverdriver/riverdatabasesql" + "github.com/riverqueue/river/riverdriver/rivermysql" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riversqlite" "github.com/riverqueue/river/rivershared/riversharedtest" @@ -188,6 +190,26 @@ func TestClientWithDriverRiverLibSQL(t *testing.T) { ) } +func TestClientWithDriverRiverMySQL(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = context.Background() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = rivermysql.New(dbPool) + ) + + ExerciseClient(ctx, t, + func(ctx context.Context, t *testing.T) (riverdriver.Driver[*sql.Tx], string) { + t.Helper() + + return driver, riverdbtest.TestSchema(ctx, t, driver, nil) + }, + ) +} + func TestClientWithDriverRiverSQLiteModernC(t *testing.T) { t.Parallel() @@ -439,7 +461,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, Schema: bundle.schema, }) require.NoError(t, getErr) - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if bundle.driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -458,7 +480,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, {Args: noOpArgs{Name: "same"}, InsertOpts: &river.InsertOpts{UniqueOpts: river.UniqueOpts{ByArgs: true}}}, {Args: noOpArgs{Name: "same"}, InsertOpts: &river.InsertOpts{UniqueOpts: river.UniqueOpts{ByArgs: true}}}, }) - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if bundle.driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -697,6 +719,39 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("JobDeleteBeforeSkipsLocked", func(t *testing.T) { + t.Parallel() + + _, bundle := setup(t) + if bundle.driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL's cleaner skips locked expired jobs to avoid secondary-index deadlocks") + } + now := time.Now().UTC() + running := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{Schema: bundle.schema, State: new(rivertype.JobStateRunning)}) + locked := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + unlocked := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + // A full batch must also avoid a DELETE table scan through locked rows. + for range 999 { + testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + } + _, execTx := beginTx(ctx, t, bundle) + for _, id := range []int64{running.ID, locked.ID} { + _, err := execTx.JobUpdate(ctx, &riverdriver.JobUpdateParams{ID: id, Metadata: []byte(`{"locked":true}`), MetadataDoMerge: true, Schema: bundle.schema}) + require.NoError(t, err) + } + cleanupCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + deleted, err := bundle.exec.JobDeleteBefore(cleanupCtx, &riverdriver.JobDeleteBeforeParams{CompletedDoDelete: true, CompletedFinalizedAtHorizon: now, Max: 1000, Schema: bundle.schema}) + require.NoError(t, err) + require.Equal(t, 1000, deleted) + _, err = bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: unlocked.ID, Schema: bundle.schema}) + require.ErrorIs(t, err, rivertype.ErrNotFound) + for _, id := range []int64{running.ID, locked.ID} { + _, err := bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: id, Schema: bundle.schema}) + require.NoError(t, err) + } + }) + t.Run("JobDeleteManyUnsafeAll", func(t *testing.T) { t.Parallel() @@ -1152,6 +1207,56 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, require.Equal(t, job.ID, listRes.Jobs[0].ID) }) + t.Run("JobListMetadataStructure", func(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + metadata string + fragment string + matches bool + }{ + {"ArrayNestingMatches", `{"array":[[1,2]]}`, `{"array":[[1]]}`, true}, + {"ArrayNestingMismatch", `{"array":[[1]]}`, `{"array":[1]}`, false}, + {"ArrayObjectMatches", `{"array":[{"a":1,"b":2}]}`, `{"array":[{"b":2,"a":1}]}`, true}, + {"ArrayObjectSplit", `{"array":[{"a":1},{"b":2}]}`, `{"array":[{"a":1,"b":2}]}`, false}, + {"ArrayScalarMismatch", `{"array":[1]}`, `{"array":1}`, false}, + {"EmptyArrayMatches", `{"array":[1]}`, `{"array":[]}`, true}, + {"EmptyContainerMismatch", `{"array":{}}`, `{"array":[]}`, false}, + {"LargeIntegerMatches", `{"number":9007199254740993}`, `{"number":9007199254740993}`, true}, + {"LargeIntegerMismatch", `{"number":9007199254740993}`, `{"number":9007199254740992}`, false}, + {"NullMatches", `{"value":null}`, `{"value":null}`, true}, + {"NullMissing", `{}`, `{"value":null}`, false}, + {"QuotedKey", `{"a.b\\c\"d'e":true}`, `{"a.b\\c\"d'e":true}`, true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + client, bundle := setup(t) + if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("Metadata containment is not supported on SQLite") + } + job := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{ + Metadata: []byte(testCase.metadata), + Schema: bundle.schema, + }) + params := river.NewJobListParams().Metadata(testCase.fragment) + result, err := client.JobList(ctx, params) + require.NoError(t, err) + wantIDs := []int64{} + if testCase.matches { + wantIDs = []int64{job.ID} + } + require.Equal(t, wantIDs, sliceutil.Map(result.Jobs, func(job *rivertype.JobRow) int64 { return job.ID })) + + tx, _ := beginTx(ctx, t, bundle) + result, err = client.JobListTx(ctx, tx, params) + require.NoError(t, err) + require.Equal(t, wantIDs, sliceutil.Map(result.Jobs, func(job *rivertype.JobRow) int64 { return job.ID })) + }) + } + }) + t.Run("JobListMixedStatePagination", func(t *testing.T) { t.Parallel() @@ -1562,9 +1667,12 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, listParams := river.NewJobListParams() - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + switch bundle.driver.DatabaseName() { + case riverdriver.DatabaseNameSQLite: listParams = listParams.Where("metadata ->> @json_path = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": "bar"}) - } else { + case riverdriver.DatabaseNameMySQL: + listParams = listParams.Where("JSON_UNQUOTE(JSON_EXTRACT(metadata, @json_path)) = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": "bar"}) + default: // "bar" is quoted in this branch because `jsonb_path_query_first` needs to be compared to a JSON value listParams = listParams.Where("jsonb_path_query_first(metadata, @json_path) = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": `"bar"`}) } @@ -1882,7 +1990,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, // The invalid value is left in place, and once repaired, the job // shows the failed attempt. require.Equal(t, sqliteMalformedValue, sqliteJobColumnText(ctx, t, bundle.exec, jobID, column)) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, jobID, column, repairedValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, jobID, column, repairedValue) } job, err := client.JobGet(ctx, jobID) @@ -1909,8 +2017,8 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, t.Parallel() config, bundle := setupConfig(t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } config.RetryPolicy = &retryPolicyAnHourLater{} @@ -1924,8 +2032,8 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, goodJob2 = testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{Kind: new(noOpArgs{}.Kind()), Schema: bundle.schema}) undecodableValue = `{"not":"an array"}` ) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob1.ID, "tags", undecodableValue) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob2.ID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob1.ID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob2.ID, "tags", undecodableValue) subscribeChan := subscribe(t, client) startClient(ctx, t, client) @@ -1951,7 +2059,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, require.ErrorContains(t, err, "error unmarshaling `tags`") // Once repaired, the job shows the failed attempt. - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob1.ID, "tags", `["tag"]`) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob1.ID, "tags", `["tag"]`) job, err := client.JobGet(ctx, undecodableJob1.ID) require.NoError(t, err) require.Equal(t, 1, job.Attempt) diff --git a/riverdriver/riverdrivertest/driver_test.go b/riverdriver/riverdrivertest/driver_test.go index 80abb865d..cb09a1643 100644 --- a/riverdriver/riverdrivertest/driver_test.go +++ b/riverdriver/riverdrivertest/driver_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + _ "github.com/go-sql-driver/mysql" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" @@ -22,6 +23,7 @@ import ( "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/riverdriver/riverdatabasesql" "github.com/riverqueue/river/riverdriver/riverdrivertest" + "github.com/riverqueue/river/riverdriver/rivermysql" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riversqlite" "github.com/riverqueue/river/rivershared/riversharedtest" @@ -234,6 +236,43 @@ func TestDriverRiverLiteLibSQL(t *testing.T) { //nolint:dupl }) } +func TestDriverRiverMySQL(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = context.Background() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = rivermysql.New(dbPool) + ) + + riverdrivertest.Exercise(ctx, t, + func(ctx context.Context, t *testing.T, opts *riverdbtest.TestSchemaOpts) (riverdriver.Driver[*sql.Tx], string) { + t.Helper() + + return driver, riverdbtest.TestSchema(ctx, t, driver, opts) + }, + func(ctx context.Context, t *testing.T) (riverdriver.Executor, riverdriver.Driver[*sql.Tx]) { + t.Helper() + + tx, schema := riverdbtest.TestTx(ctx, t, driver, &riverdbtest.TestTxOpts{ + // Disable schema sharing to reduce InnoDB deadlocks from + // parallel tests contending on the same database. + DisableSchemaSharing: true, + }) + + // MySQL has no search_path equivalent, so USE the test schema + // database so that unqualified queries resolve correctly. + if schema != "" { + _, err := tx.ExecContext(ctx, "USE "+schema) + require.NoError(t, err) + } + + return driver.UnwrapExecutor(tx), driver + }) +} + func TestDriverRiverSQLiteModernC(t *testing.T) { //nolint:dupl t.Parallel() diff --git a/riverdriver/riverdrivertest/executor_tx.go b/riverdriver/riverdrivertest/executor_tx.go index 65e1372db..4115a1f35 100644 --- a/riverdriver/riverdrivertest/executor_tx.go +++ b/riverdriver/riverdrivertest/executor_tx.go @@ -212,7 +212,13 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, exec := setup(ctx, t) - require.NoError(t, exec.Exec(ctx, "SELECT $1 || $2", "foo", "bar")) + _, driver := executorWithTx(ctx, t) + switch driver.DatabaseName() { + case riverdriver.DatabaseNameMySQL: + require.NoError(t, exec.Exec(ctx, "SELECT CONCAT(?, ?)", "foo", "bar")) + default: + require.NoError(t, exec.Exec(ctx, "SELECT $1 || $2", "foo", "bar")) + } }) }) @@ -221,9 +227,8 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, { driver, _ := driverWithSchema(ctx, t, nil) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - t.Logf("Skipping PGAdvisoryXactLock test for SQLite") - return + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite || driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + t.Skipf("Skipping PGAdvisoryXactLock test for %s", driver.DatabaseName()) } } @@ -290,6 +295,44 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, require.Equal(t, "foo", fieldFoo) }) + t.Run("QueueMutationsWithExistingSnapshot", func(t *testing.T) { + t.Parallel() + + for _, operation := range []string{"Pause", "Resume", "Update"} { + t.Run(operation, func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite does not permit a second writer with an open read transaction") + } + exec := driver.GetExecutor() + queue := testfactory.Queue(ctx, t, exec, &testfactory.QueueOpts{Schema: schema}) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.QueueGet(ctx, &riverdriver.QueueGetParams{Name: queue.Name, Schema: schema}) + require.NoError(t, err) + + deleted, err := exec.QueueDeleteExpired(ctx, &riverdriver.QueueDeleteExpiredParams{ + Max: 10, Schema: schema, UpdatedAtHorizon: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + require.Equal(t, []string{queue.Name}, deleted) + + switch operation { + case "Pause": + err = tx.QueuePause(ctx, &riverdriver.QueuePauseParams{Name: queue.Name, Schema: schema}) + case "Resume": + err = tx.QueueResume(ctx, &riverdriver.QueueResumeParams{Name: queue.Name, Schema: schema}) + case "Update": + _, err = tx.QueueUpdate(ctx, &riverdriver.QueueUpdateParams{Metadata: []byte(`{}`), Name: queue.Name, Schema: schema}) + } + require.ErrorIs(t, err, rivertype.ErrNotFound) + }) + } + }) + t.Run("WithTx", func(t *testing.T) { t.Parallel() @@ -371,6 +414,32 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, }) } + t.Run("Savepoints", func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("uses MySQL session statistics to detect hidden savepoints") + } + tx, err := driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + + var name string + var before, after int + require.NoError(t, tx.QueryRow(ctx, "SHOW SESSION STATUS LIKE 'Com_savepoint'").Scan(&name, &before)) + require.NoError(t, dbutil.WithTx(ctx, tx, func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + _ = testfactory.Job(ctx, t, execTx, &testfactory.JobOpts{Schema: schema}) + return nil + })) + require.NoError(t, tx.QueryRow(ctx, "SHOW SESSION STATUS LIKE 'Com_savepoint'").Scan(&name, &after)) + if legacySubtransactions { + require.Greater(t, after, before, "legacy mode creates savepoints") + } else { + require.Equal(t, before, after, "driver internals must reuse the caller's transaction without savepoints") + } + }) + t.Run("TransactionIDs", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/riverdrivertest/go.mod b/riverdriver/riverdrivertest/go.mod index ca51d7c86..7c28f385e 100644 --- a/riverdriver/riverdrivertest/go.mod +++ b/riverdriver/riverdrivertest/go.mod @@ -6,12 +6,14 @@ toolchain go1.26.6 require ( github.com/davecgh/go-spew v1.1.1 + github.com/go-sql-driver/mysql v1.10.0 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pgx/v5 v5.11.0 github.com/lib/pq v1.12.3 github.com/riverqueue/river v0.49.0 github.com/riverqueue/river/riverdriver v0.49.0 github.com/riverqueue/river/riverdriver/riverdatabasesql v0.49.0 + github.com/riverqueue/river/riverdriver/rivermysql v0.0.0-00010101000000-000000000000 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.49.0 github.com/riverqueue/river/riverdriver/riversqlite v0.49.0 github.com/riverqueue/river/rivershared v0.49.0 @@ -25,7 +27,12 @@ require ( turso.tech/database/tursogo v0.8.2 ) +// Remove this replacement after rivermysql has had its first coordinated +// release. It lets go mod tidy resolve the new sibling module in the meantime. +replace github.com/riverqueue/river/riverdriver/rivermysql => ../rivermysql + require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/coder/websocket v1.8.12 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/riverdriver/riverdrivertest/go.sum b/riverdriver/riverdrivertest/go.sum index 13b5d345f..24ab92b15 100644 --- a/riverdriver/riverdrivertest/go.sum +++ b/riverdriver/riverdrivertest/go.sum @@ -1,3 +1,5 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= @@ -9,6 +11,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/riverdriver/riverdrivertest/job_delete.go b/riverdriver/riverdrivertest/job_delete.go index dd8a14dd2..86084584f 100644 --- a/riverdriver/riverdrivertest/job_delete.go +++ b/riverdriver/riverdrivertest/job_delete.go @@ -137,6 +137,31 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT afterHorizon = horizon.Add(1 * time.Minute) ) + t.Run("RespectsDisabledStates", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + cancelledJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) + completedJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) + discardedJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) + + numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ + CompletedDoDelete: true, + CompletedFinalizedAtHorizon: horizon, + Max: 1_000, + }) + require.NoError(t, err) + require.Equal(t, 1, numDeleted) + + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: cancelledJob.ID}) + require.NoError(t, err) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: completedJob.ID}) + require.ErrorIs(t, err, rivertype.ErrNotFound) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: discardedJob.ID}) + require.NoError(t, err) + }) + t.Run("Success", func(t *testing.T) { t.Parallel() @@ -235,16 +260,17 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT exec, _ := setup(ctx, t) var ( - cancelledJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) - completedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) - discardedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) - excludedQueue1 = "excluded1" excludedQueue2 = "excluded2" - // Not deleted because in an omitted queue. + // Insert excluded jobs first to verify that they don't consume the + // limited candidate batch and starve later eligible jobs. notDeletedJob1 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &excludedQueue1, State: new(rivertype.JobStateCompleted)}) notDeletedJob2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &excludedQueue2, State: new(rivertype.JobStateCompleted)}) + + cancelledJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) + completedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) + discardedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) ) numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ @@ -254,7 +280,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT CompletedFinalizedAtHorizon: horizon, DiscardedDoDelete: true, DiscardedFinalizedAtHorizon: horizon, - Max: 1_000, + Max: 3, QueuesExcluded: []string{excludedQueue1, excludedQueue2}, }) require.NoError(t, err) @@ -373,6 +399,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT deletedJob2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &includedQueue2, State: new(rivertype.JobStateCompleted)}) ) + // The three older non-included jobs must not consume this batch. numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ CancelledDoDelete: true, CancelledFinalizedAtHorizon: horizon, @@ -380,7 +407,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT CompletedFinalizedAtHorizon: horizon, DiscardedDoDelete: true, DiscardedFinalizedAtHorizon: horizon, - Max: 1_000, + Max: 2, QueuesIncluded: []string{includedQueue1, includedQueue2}, }) require.NoError(t, err) @@ -550,5 +577,30 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT require.NoError(t, err) require.Equal(t, []int64{job1.ID, job2.ID, job3.ID, job4.ID, job5.ID}, sliceutil.Map(deletedJobs, func(j *rivertype.JobRow) int64 { return j.ID })) }) + + t.Run("SortedResultsDescending", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite always returns JobDeleteMany results ordered by ID") + } + + var ( + job1 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job3 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job4 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job5 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + ) + + deletedJobs, err := exec.JobDeleteMany(ctx, &riverdriver.JobDeleteManyParams{ + Max: 100, + OrderByClause: "id DESC", + WhereClause: "true", + }) + require.NoError(t, err) + require.Equal(t, []int64{job5.ID, job4.ID, job3.ID, job2.ID, job1.ID}, sliceutil.Map(deletedJobs, func(j *rivertype.JobRow) int64 { return j.ID })) + }) }) } diff --git a/riverdriver/riverdrivertest/job_insert.go b/riverdriver/riverdrivertest/job_insert.go index 172f2b971..690e0944f 100644 --- a/riverdriver/riverdrivertest/job_insert.go +++ b/riverdriver/riverdrivertest/job_insert.go @@ -93,9 +93,8 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, uniqueNonces[nonce] = true } - // SQLite needs to set a special metadata key to be able to - // check for duplicates. Remove this for purposes of comparing - // inserted metadata. + // Some drivers use a special metadata key to detect duplicates. + // Remove it for purposes of comparing inserted metadata. job.Metadata, err = sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) require.NoError(t, err) @@ -120,6 +119,33 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("DuplicateIDWithoutUniqueKeyErrors", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + id := rand.Int64() + params := &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{ + { + EncodedArgs: []byte(`{}`), + ID: &id, + Kind: "test_kind", + MaxAttempts: rivercommon.MaxAttemptsDefault, + Priority: rivercommon.PriorityDefault, + Queue: rivercommon.QueueDefault, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + }, + } + + _, err := exec.JobInsertFastMany(ctx, params) + require.NoError(t, err) + + _, err = exec.JobInsertFastMany(ctx, params) + require.Error(t, err) + }) + t.Run("MissingValuesDefaultAsExpected", func(t *testing.T) { t.Parallel() @@ -186,6 +212,62 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("MixedNewAndDuplicateJobs", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + params := &riverdriver.JobInsertFastManyParams{} + for i := range 6 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Kind: "test_kind", MaxAttempts: 25, Metadata: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}, + }) + } + for _, i := range []byte{0, 2, 4} { + // Include bytes that aren't UTF-8 and differ only after a NUL. + params.Jobs[i].UniqueKey = []byte{0xff, 0x00, i, 0x80} + params.Jobs[i].UniqueStates = 0xff + } + params.Jobs[3].UniqueKey = params.Jobs[0].UniqueKey + params.Jobs[3].UniqueStates = 1 << 2 // Inactive for an available job. + + // Seed in reverse order, so returning rows sorted by ID would be wrong. + seed, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{params.Jobs[4], params.Jobs[0]}, + }) + require.NoError(t, err) + results, err := exec.JobInsertFastMany(ctx, params) + require.NoError(t, err) + require.Len(t, results, len(params.Jobs)) + require.Equal(t, seed[1].Job.ID, results[0].Job.ID) + require.Equal(t, seed[0].Job.ID, results[4].Job.ID) + for i, result := range results { + require.Equal(t, i == 0 || i == 4, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + require.Equal(t, params.Jobs[i].UniqueKey, result.Job.UniqueKey) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: result.Job.ID}) + require.NoError(t, err) + for _, job := range []*rivertype.JobRow{result.Job, stored} { + metadata, err := sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) + require.NoError(t, err) + require.JSONEq(t, string(params.Jobs[i].Metadata), string(metadata)) + } + } + + // No new IDs are generated when every job conflicts. + duplicates, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{params.Jobs[4], params.Jobs[2], params.Jobs[0]}, + }) + require.NoError(t, err) + require.Len(t, duplicates, 3) + for i, original := range []int{4, 2, 0} { + require.True(t, duplicates[i].UniqueSkippedAsDuplicate) + require.Equal(t, results[original].Job.ID, duplicates[i].Job.ID) + } + }) + t.Run("UniqueConflict", func(t *testing.T) { t.Parallel() @@ -300,7 +382,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, Jobs: []*riverdriver.JobInsertFastParams{job, job}, Schema: schema, }) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -449,6 +531,39 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("BatchWithoutUniqueKeys", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + now := time.Date(2026, 10, 15, 12, 30, 1, 123456000, time.FixedZone("offset", -5*60*60)) + params := &riverdriver.JobInsertFastManyParams{} + for i := range 3 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + CreatedAt: &now, EncodedArgs: []byte(`{"id":123456789,"text":"é🌊\n\"quoted\""}`), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`{"id":123456789}`), + Priority: i + 1, Queue: "default", ScheduledAt: &now, + State: rivertype.JobStateAvailable, Tags: []string{"é🌊", `"quoted"`}, + }) + } + + count, err := exec.JobInsertFastManyNoReturning(ctx, params) + require.NoError(t, err) + require.Equal(t, len(params.Jobs), count) + jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{Kind: []string{"batch"}}) + require.NoError(t, err) + require.Len(t, jobs, len(params.Jobs)) + for i, job := range jobs { + require.Contains(t, string(job.EncodedArgs), "123456789") + require.Contains(t, string(job.Metadata), "123456789") + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(job.EncodedArgs)) + require.Equal(t, params.Jobs[i].Tags, job.Tags) + require.Equal(t, i+1, job.Priority) + require.WithinDuration(t, now, job.CreatedAt, bundle.driver.TimePrecision()) + require.WithinDuration(t, now, job.ScheduledAt, bundle.driver.TimePrecision()) + require.Nil(t, job.UniqueKey) + } + }) + t.Run("MissingValuesDefaultAsExpected", func(t *testing.T) { t.Parallel() @@ -837,7 +952,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, _, err := exec.JobInsertFull(ctx, params) require.Error(t, err) // two separate error messages here for Postgres and SQLite - require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null")`, err.Error()) + require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null"|Check constraint 'finalized_or_finalized_at_null' is violated)`, err.Error()) }) t.Run(fmt.Sprintf("CanSetState%sWithFinalizedAt", capitalizeJobState(state)), func(t *testing.T) { @@ -887,7 +1002,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, })) require.Error(t, err) // two separate error messages here for Postgres and SQLite - require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null")`, err.Error()) + require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null"|Check constraint 'finalized_or_finalized_at_null' is violated)`, err.Error()) }) } }) diff --git a/riverdriver/riverdrivertest/job_read.go b/riverdriver/riverdrivertest/job_read.go index 05037a26a..72dc46f4b 100644 --- a/riverdriver/riverdrivertest/job_read.go +++ b/riverdriver/riverdrivertest/job_read.go @@ -278,6 +278,8 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx exec, _ := setup(ctx, t) unknown := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("unknown"), Priority: new(1)}) + unknownCase := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("KNOWN1"), Priority: new(1)}) + unknownSpace := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1 "), Priority: new(1)}) known1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1"), Priority: new(2)}) known2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known2"), Priority: new(3)}) otherQueue := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1"), Priority: new(1), Queue: new("other")}) @@ -298,7 +300,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx require.Equal(t, rivertype.JobStateRunning, jobs[0].State) } - for _, expected := range []*rivertype.JobRow{unknown, otherQueue} { + for _, expected := range []*rivertype.JobRow{unknown, unknownCase, unknownSpace, otherQueue} { job, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: expected.ID}) require.NoError(t, err) require.Equal(t, expected, job) @@ -622,8 +624,8 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } job1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) @@ -631,9 +633,9 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx job3 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) job4 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "errors", `{"not":"an array"}`) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "tags", `{"not":"an array"}`) - sqliteSetJobJSONColumn(ctx, t, exec, job4.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "errors", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job4.ID, "tags", `{"not":"an array"}`) res, err := exec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ ClientID: testClientID, @@ -725,6 +727,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx job2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("kind2")}) // Not returned. + _ = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("KIND1")}) _ = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("kind3")}) jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{ @@ -902,8 +905,8 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } var ( @@ -914,7 +917,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx stuckJob1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &beforeHorizon, State: new(rivertype.JobStateRunning)}) stuckJob2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &beforeHorizon, State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, stuckJob1.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", stuckJob1.ID, "tags", `{"not":"an array"}`) stuckJobs, err := exec.JobGetStuck(ctx, &riverdriver.JobGetStuckParams{ Max: 10, diff --git a/riverdriver/riverdrivertest/job_update.go b/riverdriver/riverdrivertest/job_update.go index a4de886f8..e28ba5807 100644 --- a/riverdriver/riverdrivertest/job_update.go +++ b/riverdriver/riverdrivertest/job_update.go @@ -367,15 +367,15 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold a non-array errors value") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime attemptedAt := now.Add(-time.Hour) badJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &attemptedAt, State: new(rivertype.JobStateRunning)}) goodJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &attemptedAt, State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, badJob.ID, "errors", `{"error":"existing value"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", badJob.ID, "errors", `{"error":"existing value"}`) _, err := exec.JobRescueMany(ctx, &riverdriver.JobRescueManyParams{ ID: []int64{badJob.ID, goodJob.ID}, @@ -859,8 +859,8 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } var ( @@ -879,7 +879,7 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT undecodableJobIDs = []int64{undecodableJob.ID, conflictingJob.ID, scheduledJob.ID} ) for _, jobID := range undecodableJobIDs { - sqliteSetJobJSONColumn(ctx, t, exec, jobID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", jobID, "tags", undecodableValue) } // Conflicts with conflictingJob, which is discarded instead of scheduled. @@ -1173,14 +1173,14 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold a non-array errors value") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, job.ID, "errors", `{"error":"existing value"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job.ID, "errors", `{"error":"existing value"}`) jobsAfter, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams(riverdriver.JobSetStateErrorRetryable(job.ID, now, makeErrPayload(t, now), nil))) require.NoError(t, err) @@ -1254,15 +1254,15 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime job1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) job2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "tags", `{"not":"an array"}`) jobsAfter, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams( riverdriver.JobSetStateErrorRetryable(job1.ID, now, makeErrPayload(t, now), nil), @@ -1489,6 +1489,75 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT }) }) + t.Run("JobSetStateIfRunningMany_MetadataDoesNotChangeCancellationDecision", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) + cancelled := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Attempt: new(3), Metadata: []byte(`{"cancel_attempted_at":"already requested"}`), + State: new(rivertype.JobStateRunning), + }) + completed := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + FinalizedAt: &precisionTestTime, State: new(rivertype.JobStateCompleted), + }) + rows, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams( + riverdriver.JobSetStateErrorRetryable(job.ID, precisionTestTime, []byte(`{"error":"retry"}`), []byte(`{"cancel_attempted_at":"from metadata update"}`)), + riverdriver.JobSetStateSnoozed(cancelled.ID, precisionTestTime.Add(time.Hour), 2, []byte(`{"cancel_attempted_at":null}`)), + riverdriver.JobSetStateErrorRetryable(completed.ID, precisionTestTime.Add(time.Hour), []byte(`{"error":"ignored"}`), []byte(`{"result":"merged"}`)), + )) + require.NoError(t, err) + require.Len(t, rows, 3) + require.Equal(t, rivertype.JobStateRetryable, rows[0].State) + require.Nil(t, rows[0].FinalizedAt) + require.Equal(t, "from metadata update", gjson.GetBytes(rows[0].Metadata, "cancel_attempted_at").String()) + require.Equal(t, rivertype.JobStateCancelled, rows[1].State) + require.NotNil(t, rows[1].FinalizedAt) + require.Equal(t, cancelled.Attempt, rows[1].Attempt) + require.Equal(t, cancelled.ScheduledAt, rows[1].ScheduledAt) + require.Empty(t, gjson.GetBytes(rows[1].Metadata, "cancel_attempted_at").String()) + require.Equal(t, rivertype.JobStateCompleted, rows[2].State) + require.Equal(t, completed.FinalizedAt, rows[2].FinalizedAt) + require.Equal(t, completed.ScheduledAt, rows[2].ScheduledAt) + require.Empty(t, rows[2].Errors) + require.JSONEq(t, `{"result":"merged"}`, string(rows[2].Metadata)) + }) + + t.Run("JobSetStateIfRunningMany_MetadataMergeIsShallow", func(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"Batch", "RepeatedID", "Single"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + if mode == "RepeatedID" && bundle.driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL retains sequential updates for repeated IDs") + } + now := time.Now().UTC() + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Metadata: []byte(`{"checkpoint":"before","keep":true,"nested":{"old":1}}`), + State: new(rivertype.JobStateRunning), + }) + updates := []*riverdriver.JobSetStateIfRunningParams{riverdriver.JobSetStateCompleted(job.ID, now, []byte(`{"checkpoint":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`))} + switch mode { + case "Batch": + other := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) + updates = append(updates, riverdriver.JobSetStateCompleted(other.ID, now, []byte(`{"checkpoint":null}`))) + case "RepeatedID": + updates = append(updates, riverdriver.JobSetStateCompleted(job.ID, now, []byte(`{"keep":true}`))) + } + _, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams(updates...)) + require.NoError(t, err) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: job.ID}) + require.NoError(t, err) + require.JSONEq(t, `{"checkpoint":null,"keep":true,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`, string(stored.Metadata)) + require.Equal(t, "9007199254740993", gjson.GetBytes(stored.Metadata, "large").Raw) + }) + } + }) + t.Run("JobSetStateIfRunningMany_MultipleJobsAtOnce", func(t *testing.T) { t.Parallel() @@ -1538,16 +1607,16 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT exec, _ := setup(ctx, t) job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ - Metadata: []byte(`{"key1":"val1"}`), + Metadata: []byte(`{"key1":"val1","nested":{"old":1}}`), }) updatedJob, err := exec.JobUpdate(ctx, &riverdriver.JobUpdateParams{ ID: job.ID, MetadataDoMerge: true, - Metadata: []byte(`{"key2":"val2"}`), + Metadata: []byte(`{"key2":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`), }) require.NoError(t, err) - require.JSONEq(t, `{"key1":"val1","key2":"val2"}`, string(updatedJob.Metadata)) + require.JSONEq(t, `{"key1":"val1","key2":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`, string(updatedJob.Metadata)) }) t.Run("NoArgs", func(t *testing.T) { diff --git a/riverdriver/riverdrivertest/leader.go b/riverdriver/riverdrivertest/leader.go index 7a6cf55f4..57b6283ba 100644 --- a/riverdriver/riverdrivertest/leader.go +++ b/riverdriver/riverdrivertest/leader.go @@ -211,6 +211,29 @@ func exerciseLeader[TTx any](ctx context.Context, t *testing.T, executorWithTx f require.Equal(t, leader.ElectedAt, leaderFromDB.ElectedAt) }) + t.Run("ReelectsWithoutChangingExpiry", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + + now := time.Now().UTC().Truncate(bundle.driver.TimePrecision()) + leader, err := exec.LeaderAttemptElect(ctx, &riverdriver.LeaderElectParams{ + LeaderID: testClientID, + Now: &now, + TTL: leaderTTL, + }) + require.NoError(t, err) + + updatedLeader, err := exec.LeaderAttemptReelect(ctx, &riverdriver.LeaderReelectParams{ + ElectedAt: leader.ElectedAt, + LeaderID: testClientID, + Now: &now, + TTL: leaderTTL, + }) + require.NoError(t, err) + require.Equal(t, leader, updatedLeader) + }) + t.Run("WithoutNow", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/riverdrivertest/listener.go b/riverdriver/riverdrivertest/listener.go index 6b8df6de8..9404b16d2 100644 --- a/riverdriver/riverdrivertest/listener.go +++ b/riverdriver/riverdrivertest/listener.go @@ -235,8 +235,8 @@ func exerciseListener[TTx any](ctx context.Context, t *testing.T, driverWithPool listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: ""}) ) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - t.Skip("SQLite has no search_path") + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skipf("%s has no search_path", driver.DatabaseName()) } listener.SetAfterConnectExec("SET search_path TO 'public'") @@ -253,6 +253,10 @@ func exerciseListener[TTx any](ctx context.Context, t *testing.T, driverWithPool listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: ""}) ) + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + t.Skip("MySQL has no search_path") + } + connectListener(ctx, t, listener) require.Empty(t, listener.Schema()) }) diff --git a/riverdriver/riverdrivertest/migration.go b/riverdriver/riverdrivertest/migration.go index bf37e18b1..81d765cb0 100644 --- a/riverdriver/riverdrivertest/migration.go +++ b/riverdriver/riverdrivertest/migration.go @@ -253,6 +253,11 @@ func exerciseMigration[TTx any](ctx context.Context, t *testing.T, t.Run("MigrateVersionEightRejectsProSchema", func(t *testing.T) { t.Parallel() + driver, _ := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() != riverdriver.DatabaseNameSQLite { + t.Skip("SQLite table rebuild") + } + for _, testCase := range []struct { name string sql string @@ -279,9 +284,6 @@ func exerciseMigration[TTx any](ctx context.Context, t *testing.T, }, Lines: []string{riverdriver.MigrationLineMain}, }) - if driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("SQLite table rebuild") - } exec := driver.GetExecutor() job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Schema: schema}) diff --git a/riverdriver/riverdrivertest/notification.go b/riverdriver/riverdrivertest/notification.go index 2f1025084..fbb547bf5 100644 --- a/riverdriver/riverdrivertest/notification.go +++ b/riverdriver/riverdrivertest/notification.go @@ -31,7 +31,7 @@ func exerciseNotification[TTx any](ctx context.Context, t *testing.T, executorWi ($7, $8, $9), ($10, $11, $12) ` - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { insertQuery = ` INSERT INTO river_notification (created_at, payload, topic) VALUES diff --git a/riverdriver/riverdrivertest/riverdrivertest.go b/riverdriver/riverdrivertest/riverdrivertest.go index b211fe7af..e3f943fc7 100644 --- a/riverdriver/riverdrivertest/riverdrivertest.go +++ b/riverdriver/riverdrivertest/riverdrivertest.go @@ -100,6 +100,25 @@ func exerciseDriverPool[TTx any](ctx context.Context, t *testing.T, }) }) + t.Run("SafeIdentifier", func(t *testing.T) { + t.Parallel() + + _, driver := executorWithTx(ctx, t) + + switch driver.DatabaseName() { + case riverdriver.DatabaseNamePostgres, riverdriver.DatabaseNameSQLite: + require.Equal(t, `"my_schema"`, driver.SafeIdentifier("my_schema")) + require.Equal(t, `"has space"`, driver.SafeIdentifier("has space")) + require.Equal(t, `"has""quote"`, driver.SafeIdentifier(`has"quote`)) + case riverdriver.DatabaseNameMySQL: + require.Equal(t, "`my_schema`", driver.SafeIdentifier("my_schema")) + require.Equal(t, "`has space`", driver.SafeIdentifier("has space")) + require.Equal(t, "`has``backtick`", driver.SafeIdentifier("has`backtick")) + default: + require.FailNow(t, "Don't know how to check SafeIdentifier for: "+driver.DatabaseName()) + } + }) + t.Run("SupportsListenNotify", func(t *testing.T) { t.Parallel() @@ -110,6 +129,8 @@ func exerciseDriverPool[TTx any](ctx context.Context, t *testing.T, require.True(t, driver.SupportsListenNotify()) case riverdriver.DatabaseNameSQLite: require.True(t, driver.SupportsListenNotify()) + case riverdriver.DatabaseNameMySQL: + require.True(t, driver.SupportsListenNotify()) default: require.FailNow(t, "Don't know how to check SupportsListenNotify for: "+driver.DatabaseName()) } @@ -126,7 +147,8 @@ func requireMissingRelation(t *testing.T, err error, schema, missingRelation str // lib/pq: pq: relation %s.%s does not exist // SQLite: no such table: %s.%s // Turso: turso: error: Invalid argument supplied: no such database: %s - require.Regexp(t, fmt.Sprintf(`(pq: relation "%s\.%s" does not exist|no such table: %s\.%s|no such database: %s)`, schema, missingRelation, schema, missingRelation, schema), err.Error()) + // MySQL: Unknown database '%s' + require.Regexp(t, fmt.Sprintf(`(pq: relation "%s\.%s" does not exist|no such table: %s\.%s|no such database: %s|Unknown database '%s')`, schema, missingRelation, schema, missingRelation, schema, schema), err.Error()) } } @@ -140,6 +162,23 @@ var sqliteJobJSONColumns = []string{"args", "attempted_by", "errors", "metadata" // SQLite's JSON functions that touch it. const sqliteMalformedValue = "not json" +// setJobJSONColumn overwrites a JSON column of a SQLite or MySQL job row with the +// given JSON, simulating a row changed out of band into a shape that River +// can't decode. Postgres' column types don't allow the equivalent. +func setJobJSONColumn[TTx any](ctx context.Context, t *testing.T, exec riverdriver.Executor, driver riverdriver.Driver[TTx], schema string, jobID int64, column, jsonValue string) { + t.Helper() + + table := "river_job" + if schema != "" { + table = driver.SafeIdentifier(schema) + "." + table + } + valueSQL := "jsonb(?)" + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + valueSQL = "CAST(? AS JSON)" + } + require.NoError(t, exec.Exec(ctx, "UPDATE "+table+" SET "+column+" = "+valueSQL+" WHERE id = ?", jsonValue, jobID)) +} + // sqliteJobColumnText returns a column of a SQLite job row cast to text, which // can be used to check that a value that isn't valid JSON (and so can't be // read through the driver) was left in place. @@ -160,12 +199,3 @@ func sqliteSetJobColumnMalformed(ctx context.Context, t *testing.T, exec riverdr require.NoError(t, exec.Exec(ctx, "UPDATE river_job SET "+column+" = ? WHERE id = ?", sqliteMalformedValue, jobID)) } - -// sqliteSetJobJSONColumn overwrites a JSON column of a SQLite job row with the -// given JSON, simulating a row changed out of band into a shape that River -// can't decode. Postgres' column types don't allow the equivalent. -func sqliteSetJobJSONColumn(ctx context.Context, t *testing.T, exec riverdriver.Executor, jobID int64, column, jsonValue string) { - t.Helper() - - require.NoError(t, exec.Exec(ctx, "UPDATE river_job SET "+column+" = jsonb(?) WHERE id = ?", jsonValue, jobID)) -} diff --git a/riverdriver/riverdrivertest/sql_fragments.go b/riverdriver/riverdrivertest/sql_fragments.go index c0686e4e9..9bce32891 100644 --- a/riverdriver/riverdrivertest/sql_fragments.go +++ b/riverdriver/riverdrivertest/sql_fragments.go @@ -8,6 +8,7 @@ import ( "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/rivershared/testfactory" + "github.com/riverqueue/river/rivertype" ) func exerciseSQLFragments[TTx any](ctx context.Context, t *testing.T, executorWithTx func(ctx context.Context, t *testing.T) (riverdriver.Executor, riverdriver.Driver[TTx])) { @@ -95,6 +96,33 @@ func exerciseSQLFragments[TTx any](ctx context.Context, t *testing.T, executorWi require.Equal(t, job2.ID, jobs[1].ID) }) + t.Run("LargeIntegerValues", func(t *testing.T) { + t.Parallel() + + exec, driver := executorWithTx(ctx, t) + + const id = int64(1 << 53) + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: []*riverdriver.JobInsertFastParams{ + {ID: new(id), EncodedArgs: []byte(`{}`), Kind: "large_id", MaxAttempts: 1, Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}}, + {ID: new(id + 1), EncodedArgs: []byte(`{}`), Kind: "large_id", MaxAttempts: 1, Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}}, + }}) + require.NoError(t, err) + + for _, filterID := range []int64{id, id + 1} { + fragment, arg, err := driver.SQLFragmentColumnIn("id", []int64{filterID}) + require.NoError(t, err) + jobs, err := exec.JobList(ctx, &riverdriver.JobListParams{ + Max: 100, + NamedArgs: map[string]any{"id": arg}, + OrderByClause: "id", + WhereClause: fragment, + }) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, filterID, jobs[0].ID) + } + }) + t.Run("StringValues", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/rivermysql/example_mysql_test.go b/riverdriver/rivermysql/example_mysql_test.go new file mode 100644 index 000000000..74aa01770 --- /dev/null +++ b/riverdriver/rivermysql/example_mysql_test.go @@ -0,0 +1,125 @@ +package rivermysql_test + +import ( + "cmp" + "context" + "database/sql" + "fmt" + "log/slog" + "os" + "sort" + + _ "github.com/go-sql-driver/mysql" + + "github.com/riverqueue/river" + "github.com/riverqueue/river/riverdriver/rivermysql" + "github.com/riverqueue/river/rivermigrate" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/util/slogutil" + "github.com/riverqueue/river/rivershared/util/testutil" +) + +type MySQLSortArgs struct { + // Strings is a slice of strings to sort. + Strings []string `json:"strings"` +} + +func (MySQLSortArgs) Kind() string { return "sort" } + +type MySQLSortWorker struct { + river.WorkerDefaults[MySQLSortArgs] +} + +func (w *MySQLSortWorker) Work(ctx context.Context, job *river.Job[MySQLSortArgs]) error { + sort.Strings(job.Args.Strings) + fmt.Printf("Sorted strings: %+v\n", job.Args.Strings) + return nil +} + +// Example_mysql demonstrates use of River's MySQL driver. +func Example_mysql() { + // MySQL tests are opt-in because they require a running server. When + // disabled, print the expected output so the example test always passes. + val := os.Getenv("RIVER_MYSQL_TESTS_ENABLED") + if val != "1" && val != "true" { + fmt.Println("Sorted strings: [bear tiger whale]") + return + } + + ctx := context.Background() + + dsn := cmp.Or( + os.Getenv("TEST_MYSQL_URL"), + "root@tcp(localhost:3306)/?parseTime=true&multiStatements=true&loc=UTC&time_zone=%27%2B00%3A00%27", + ) + + dbPool, err := sql.Open("mysql", dsn) + if err != nil { + panic(err) + } + defer dbPool.Close() + + // Create a temporary database for the example. + const exampleDB = "river_example_mysql" + if _, err := dbPool.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+exampleDB); err != nil { + panic(err) + } + defer func() { + _, _ = dbPool.ExecContext(ctx, "DROP DATABASE IF EXISTS "+exampleDB) + }() + + // Run River's migrations to prepare the schema. + migrator, err := rivermigrate.New(rivermysql.New(dbPool), &rivermigrate.Config{ + Logger: slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelWarn, ReplaceAttr: slogutil.NoLevelTime})), + Schema: exampleDB, + }) + if err != nil { + panic(err) + } + if _, err := migrator.Migrate(ctx, rivermigrate.DirectionUp, nil); err != nil { + panic(err) + } + + workers := river.NewWorkers() + river.AddWorker(workers, &MySQLSortWorker{}) + + riverClient, err := river.NewClient(rivermysql.New(dbPool), &river.Config{ + Logger: slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelWarn, ReplaceAttr: slogutil.NoLevelTime})), + Queues: map[string]river.QueueConfig{ + river.QueueDefault: {MaxWorkers: 100}, + }, + Schema: exampleDB, + TestOnly: true, // suitable only for use in tests; remove for live environments + Workers: workers, + }) + if err != nil { + panic(err) + } + + // Out of example scope, but used to wait until a job is worked. + subscribeChan, subscribeCancel := riverClient.Subscribe(river.EventKindJobCompleted) + defer subscribeCancel() + + if err := riverClient.Start(ctx); err != nil { + panic(err) + } + + _, err = riverClient.Insert(ctx, MySQLSortArgs{ + Strings: []string{ + "whale", "tiger", "bear", + }, + }, nil) + if err != nil { + panic(err) + } + + // Wait for jobs to complete. Only needed for purposes of the example test. + riversharedtest.WaitOrTimeoutN(testutil.PanicTB(), subscribeChan, 1) + + if err := riverClient.Stop(ctx); err != nil { + panic(err) + } + + // Output: + // Sorted strings: [bear tiger whale] +} diff --git a/riverdriver/rivermysql/go.mod b/riverdriver/rivermysql/go.mod new file mode 100644 index 000000000..850b57364 --- /dev/null +++ b/riverdriver/rivermysql/go.mod @@ -0,0 +1,31 @@ +module github.com/riverqueue/river/riverdriver/rivermysql + +go 1.26.0 + +toolchain go1.26.6 + +require ( + github.com/go-sql-driver/mysql v1.10.0 + github.com/riverqueue/river v0.49.0 + github.com/riverqueue/river/riverdriver v0.49.0 + github.com/riverqueue/river/rivershared v0.49.0 + github.com/riverqueue/river/rivertype v0.49.0 + github.com/stretchr/testify v1.12.1 + github.com/tidwall/gjson v1.20.0 + github.com/tidwall/sjson v1.2.5 +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.11.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.49.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + go.uber.org/goleak v1.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/text v0.42.0 // indirect +) diff --git a/riverdriver/rivermysql/go.sum b/riverdriver/rivermysql/go.sum new file mode 100644 index 000000000..968ece325 --- /dev/null +++ b/riverdriver/rivermysql/go.sum @@ -0,0 +1,54 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/riverqueue/river v0.49.0 h1:JUCLFgregbX1Wu+bSTHCjF/WHGdrwGbtz/ddInHfeb0= +github.com/riverqueue/river v0.49.0/go.mod h1:USYb57gpMBXLQm2l7poskAalOPaf5uHjSZgp2AOoYDw= +github.com/riverqueue/river/riverdriver v0.49.0 h1:kSykNQJNB7AeG6kudjB0mThV29PvykoOBIFT8ipCHEc= +github.com/riverqueue/river/riverdriver v0.49.0/go.mod h1:rVUuX/fTF2kAiJjpPT+10Lft8n90kvLqjm6b+rFXaVE= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.49.0 h1:c7YA1plP/nrNS3SEHuKizf4wHx9OQ87Yxuhj3V0eQ50= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.49.0/go.mod h1:o7zkFstM+Fk+mnlOW2sx92mQW4JswSc6zvN191l8Isg= +github.com/riverqueue/river/rivershared v0.49.0 h1:wnCYVwftMiu85kT1JrUPKEgujKkBIWoRtSNZJTUtotY= +github.com/riverqueue/river/rivershared v0.49.0/go.mod h1:E8UzQAdDutFT8rVL1wZeNbuphmIS81TgngU7/1F4U5E= +github.com/riverqueue/river/rivertype v0.49.0 h1:3up3P2DtOqnM2yEYSC1xeK9AjrljOtIfl10yZlaF4aM= +github.com/riverqueue/river/rivertype v0.49.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.20.0 h1:+agJ3rEzKcCXKDKo0ml26UROcpcAlnZyjq+TjbY5Fto= +github.com/tidwall/gjson v1.20.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/riverdriver/rivermysql/internal/dbbatch/job_insert.sql b/riverdriver/rivermysql/internal/dbbatch/job_insert.sql new file mode 100644 index 000000000..40879ba5e --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_insert.sql @@ -0,0 +1,29 @@ +INSERT INTO /* TEMPLATE: schema */river_job ( + id, args, created_at, kind, max_attempts, metadata, priority, queue, + scheduled_at, state, tags, unique_key, unique_states +) +SELECT + batch.id, CAST(batch.args AS JSON), + COALESCE(batch.created_at, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + batch.kind, batch.max_attempts, CAST(batch.metadata AS JSON), batch.priority, batch.queue, + COALESCE(batch.scheduled_at, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + batch.state, CAST(batch.tags AS JSON), UNHEX(NULLIF(batch.unique_key, '')), NULLIF(batch.unique_states, 0) +FROM JSON_TABLE(?, '$[*]' COLUMNS ( + ordinal FOR ORDINALITY, + id BIGINT PATH '$.id' ERROR ON ERROR, + args LONGTEXT CHARACTER SET utf8mb4 PATH '$.args' ERROR ON ERROR, + created_at DATETIME(6) PATH '$.created_at' ERROR ON ERROR, + kind LONGTEXT CHARACTER SET utf8mb4 PATH '$.kind' ERROR ON ERROR, + max_attempts BIGINT PATH '$.max_attempts' ERROR ON ERROR, + metadata LONGTEXT CHARACTER SET utf8mb4 PATH '$.metadata' ERROR ON ERROR, + priority BIGINT PATH '$.priority' ERROR ON ERROR, + queue LONGTEXT CHARACTER SET utf8mb4 PATH '$.queue' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.scheduled_at' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.state' ERROR ON ERROR, + tags LONGTEXT CHARACTER SET utf8mb4 PATH '$.tags' ERROR ON ERROR, + unique_active BOOLEAN PATH '$.unique_active' ERROR ON ERROR, + unique_key LONGTEXT PATH '$.unique_key' ERROR ON ERROR, + unique_states INT PATH '$.unique_states' ERROR ON ERROR +)) AS batch +ORDER BY batch.ordinal +/* TEMPLATE: on_duplicate_key */; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql b/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql new file mode 100644 index 000000000..2abfbf07c --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql @@ -0,0 +1,20 @@ +-- MySQL cannot target a particular unique index. Only accept a conflict when +-- both rows enforce River uniqueness for the same key. An invalid JSON document +-- makes unrelated conflicts fail atomically, including outside strict SQL mode. +ON DUPLICATE KEY UPDATE id = IF( + batch.unique_active + AND river_job.unique_key = UNHEX(batch.unique_key) + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1, + river_job.id, + CAST('non-River unique constraint conflict' AS JSON) +) diff --git a/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql b/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql new file mode 100644 index 000000000..6333f09e1 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql @@ -0,0 +1,26 @@ +UPDATE /* TEMPLATE: schema */river_job AS job +JOIN JSON_TABLE(?, '$[*]' COLUMNS ( + id BIGINT PATH '$.id' ERROR ON ERROR, + error LONGTEXT CHARACTER SET utf8mb4 PATH '$.error' ERROR ON ERROR, + finalized_at DATETIME(6) PATH '$.finalized_at' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.scheduled_at' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.state' ERROR ON ERROR +)) AS batch ON batch.id = job.id +SET + job.errors = JSON_ARRAY_APPEND(COALESCE(job.errors, JSON_ARRAY()), '$', CAST(batch.error AS JSON)), + job.finalized_at = batch.finalized_at, + job.scheduled_at = batch.scheduled_at, + job.metadata = JSON_SET( + job.metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(job.metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(job.metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(job.metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + job.state = batch.state +WHERE job.state = 'running' AND job.attempted_at < ?; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql b/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql new file mode 100644 index 000000000..5ec11761f --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql @@ -0,0 +1,38 @@ +-- A joined UPDATE doesn't guarantee assignment order. Materialize the old state +-- and cancellation flag before changing either state or metadata. The locking +-- read must see current values even in an existing REPEATABLE READ transaction. +UPDATE /*+ NO_MERGE(batch) */ /* TEMPLATE: schema */river_job AS job +JOIN ( + SELECT input.*, original.state = 'running' AS was_running, + input.state IN ('available', 'retryable', 'scheduled') + AND JSON_CONTAINS_PATH(original.metadata, 'one', '$.cancel_attempted_at') AS should_cancel + FROM JSON_TABLE(?, '$[*]' COLUMNS ( + id BIGINT PATH '$.id' ERROR ON ERROR, + attempt BIGINT PATH '$.attempt' ERROR ON ERROR, + error LONGTEXT CHARACTER SET utf8mb4 PATH '$.error' ERROR ON ERROR, + finalized_at DATETIME(6) PATH '$.finalized_at' ERROR ON ERROR, + metadata LONGTEXT CHARACTER SET utf8mb4 PATH '$.metadata' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.scheduled_at' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.state' ERROR ON ERROR + )) AS input + JOIN /* TEMPLATE: schema */river_job AS original ON original.id = input.id + FOR UPDATE OF original +) AS batch ON batch.id = job.id +SET + job.attempt = CASE WHEN batch.was_running AND NOT batch.should_cancel + THEN COALESCE(batch.attempt, job.attempt) ELSE job.attempt END, + job.errors = CASE WHEN batch.was_running AND batch.error IS NOT NULL + THEN JSON_ARRAY_APPEND(COALESCE(job.errors, JSON_ARRAY()), '$', CAST(batch.error AS JSON)) + ELSE job.errors END, + job.finalized_at = CASE + WHEN batch.was_running AND batch.should_cancel + THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN batch.was_running THEN COALESCE(batch.finalized_at, job.finalized_at) + ELSE job.finalized_at END, + job.scheduled_at = CASE WHEN batch.was_running AND NOT batch.should_cancel + THEN COALESCE(batch.scheduled_at, job.scheduled_at) ELSE job.scheduled_at END, + job.state = CASE + WHEN batch.was_running AND batch.should_cancel THEN 'cancelled' + WHEN batch.was_running THEN batch.state ELSE job.state END, + job.metadata = CASE WHEN batch.metadata IS NOT NULL + THEN CAST(batch.metadata AS JSON) ELSE job.metadata END; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql b/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql new file mode 100644 index 000000000..e5d042ec8 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql @@ -0,0 +1,16 @@ +-- Match the stored index expression, including its explicit string charset. +CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN _utf8mb4'available' THEN unique_states & (1 << 0) + WHEN _utf8mb4'cancelled' THEN unique_states & (1 << 1) + WHEN _utf8mb4'completed' THEN unique_states & (1 << 2) + WHEN _utf8mb4'discarded' THEN unique_states & (1 << 3) + WHEN _utf8mb4'pending' THEN unique_states & (1 << 4) + WHEN _utf8mb4'retryable' THEN unique_states & (1 << 5) + WHEN _utf8mb4'running' THEN unique_states & (1 << 6) + WHEN _utf8mb4'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END diff --git a/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql b/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql new file mode 100644 index 000000000..b17100b4a --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql @@ -0,0 +1,8 @@ +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +SELECT batch.payload, config.topic +FROM JSON_TABLE(?, '$[*]' COLUMNS ( + ordinal FOR ORDINALITY, + payload LONGTEXT CHARACTER SET utf8mb4 PATH '$' ERROR ON ERROR +)) AS batch +CROSS JOIN (SELECT ? AS topic) AS config +ORDER BY batch.ordinal; diff --git a/riverdriver/rivermysql/internal/dbfragment/json_contains.sql b/riverdriver/rivermysql/internal/dbfragment/json_contains.sql new file mode 100644 index 000000000..da4dda2d4 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbfragment/json_contains.sql @@ -0,0 +1,5 @@ +{{define "container"}}(JSON_TYPE({{.source}}) = '{{.kind}}' AND {{.children}}){{end}} +{{define "member"}}(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT({{.source}}, {{.param}}), '$' COLUMNS(value JSON PATH '$')) AS {{.alias}} WHERE {{.predicate}} LIMIT 1) IS NOT NULL{{end}} +{{define "element"}}(SELECT 1 FROM JSON_TABLE({{.source}}, '$[*]' COLUMNS(value JSON PATH '$')) AS {{.alias}} WHERE {{.predicate}} LIMIT 1) IS NOT NULL{{end}} +{{define "null"}}({{.source}} IS NULL OR JSON_TYPE({{.source}}) = 'NULL'){{end}} +{{define "scalar"}}(JSON_TYPE({{.source}}) NOT IN ('ARRAY', 'OBJECT') AND JSON_CONTAINS({{.source}}, CAST({{.param}} AS JSON))){{end}} diff --git a/riverdriver/rivermysql/internal/dbsqlc/db.go b/riverdriver/rivermysql/internal/dbsqlc/db.go new file mode 100644 index 000000000..c399d01ff --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/db.go @@ -0,0 +1,24 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package dbsqlc + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New() *Queries { + return &Queries{} +} + +type Queries struct { +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/models.go b/riverdriver/rivermysql/internal/dbsqlc/models.go new file mode 100644 index 000000000..7e3678ae1 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/models.go @@ -0,0 +1,80 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package dbsqlc + +import ( + "database/sql" + "time" +) + +type Columns struct { + ColumnName string + TableName string + TableSchema string +} + +type RiverJob struct { + ID int64 + Args []byte + Attempt int64 + AttemptedAt sql.NullTime + AttemptedBy []byte + CreatedAt time.Time + Errors []byte + FinalizedAt sql.NullTime + Kind string + MaxAttempts int64 + Metadata []byte + Priority int16 + Queue string + State string + ScheduledAt time.Time + Tags []byte + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +type RiverLeader struct { + ElectedAt time.Time + ExpiresAt time.Time + LeaderID string + Name string +} + +type RiverMigration struct { + Line string + Version int64 + CreatedAt time.Time +} + +type RiverNotification struct { + ID int64 + CreatedAt time.Time + Payload string + Topic string +} + +type RiverQueue struct { + Name string + CreatedAt time.Time + Metadata []byte + PausedAt sql.NullTime + UpdatedAt time.Time +} + +type Schemata struct { + SchemaName string +} + +type Statistics struct { + IndexName string + TableName string + TableSchema string +} + +type Tables struct { + TableName string + TableSchema string +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_job.sql b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql new file mode 100644 index 000000000..98c330c08 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql @@ -0,0 +1,469 @@ +CREATE TABLE river_job ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + args JSON NOT NULL DEFAULT (JSON_OBJECT()), + attempt INT NOT NULL DEFAULT 0, + attempted_at DATETIME(6) NULL, + attempted_by JSON NULL, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + errors JSON NULL, + finalized_at DATETIME(6) NULL, + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + max_attempts INT NOT NULL DEFAULT 25, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + priority SMALLINT NOT NULL DEFAULT 1, + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + scheduled_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + tags JSON NOT NULL DEFAULT (JSON_ARRAY()), + unique_key VARBINARY(255) NULL, + unique_states SMALLINT NULL +); + +-- name: JobGetByID :one +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') +LIMIT 1; + +-- name: JobGetByIDMany :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id; + +-- name: JobGetByIDManyForUpdate :many +-- Read current versions after writes, even in a pre-existing REPEATABLE READ transaction. +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id +FOR UPDATE; + +-- name: JobGetByInsertNonce :many +-- Own writes are visible even in an older REPEATABLE READ snapshot. Do not use +-- a locking read: the range can contain interleaved inserts from other sessions. +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id >= sqlc.arg('first_id') + AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$."river:unique_nonce"')) LIKE sqlc.arg('nonce_prefix') +ORDER BY id; + +-- name: JobGetByKindMany :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE kind IN (sqlc.slice('kind')) +ORDER BY id; + +-- name: JobGetByUniqueKeyManyForUpdate :many +-- Match the expression in river_job_unique_idx so lookups use the unique index. +-- A current read is needed for existing jobs committed after a caller's snapshot. +-- Force the index to avoid locking unrelated rows if a table scan looks cheaper. +SELECT * +FROM /* TEMPLATE: schema */river_job FORCE INDEX (river_job_unique_idx) +WHERE /* TEMPLATE_BEGIN: unique_key */ unique_key /* TEMPLATE_END */ IN (sqlc.slice('unique_keys')) +ORDER BY id +FOR UPDATE; + +-- name: JobCancelExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END, + finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) END, + metadata = JSON_SET(metadata, '$.cancel_attempted_at', CAST(sqlc.arg('cancel_attempted_at') AS CHAR)) +WHERE id = sqlc.arg('id') + AND state NOT IN ('cancelled', 'completed', 'discarded') + AND finalized_at IS NULL; + +-- name: JobClearUniqueNonce :exec +-- Avoid a table scan locking other sessions' uncommitted inserts. +UPDATE /* TEMPLATE: schema */river_job FORCE INDEX (PRIMARY) +SET metadata = JSON_REMOVE(metadata, '$."river:unique_nonce"') +WHERE id IN (sqlc.slice('id')); + +-- name: JobCountByAllStates :many +SELECT state, count(*) AS count +FROM /* TEMPLATE: schema */river_job +GROUP BY state; + +-- name: JobCountByQueueAndState :many +SELECT + queue, + COUNT(CASE WHEN state = 'available' THEN 1 END) AS count_available, + COUNT(CASE WHEN state = 'running' THEN 1 END) AS count_running +FROM /* TEMPLATE: schema */river_job +WHERE queue IN (sqlc.slice('queue_names')) + AND state IN ('available', 'running') +GROUP BY queue +ORDER BY queue; + +-- name: JobCountByState :one +SELECT count(*) AS count +FROM /* TEMPLATE: schema */river_job +WHERE state = sqlc.arg('state'); + +-- name: JobDeleteExec :execresult +DELETE FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') + AND river_job.state != 'running'; + +-- name: JobGetMetadataForUpdate :many +SELECT /*+ JOIN_INDEX(river_job PRIMARY) */ id, metadata +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id +FOR UPDATE; + +-- name: JobGetByIDForUpdate :one +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') +LIMIT 1 +FOR UPDATE; + +-- name: JobDeleteBeforeSelect :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE ( + ( + CAST(sqlc.arg('cancelled_do_delete') AS UNSIGNED) + AND state = 'cancelled' + AND finalized_at < sqlc.arg('cancelled_finalized_at_horizon') + ) OR ( + CAST(sqlc.arg('completed_do_delete') AS UNSIGNED) + AND state = 'completed' + AND finalized_at < sqlc.arg('completed_finalized_at_horizon') + ) OR ( + CAST(sqlc.arg('discarded_do_delete') AS UNSIGNED) + AND state = 'discarded' + AND finalized_at < sqlc.arg('discarded_finalized_at_horizon') + ) +) +AND ( + CAST(sqlc.arg('queues_excluded_empty') AS SIGNED) + OR queue NOT IN (sqlc.slice('queues_excluded')) +) +AND ( + CAST(sqlc.arg('queues_included_empty') AS SIGNED) + OR queue IN (sqlc.slice('queues_included')) +) +ORDER BY id +LIMIT ? +FOR UPDATE SKIP LOCKED; + +-- name: JobDeleteManySelect :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + AND state != 'running' +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +FOR UPDATE; + +-- Keep large batches on primary-key ranges so a scan cannot lock skipped jobs. +-- name: JobDeleteManyExec :exec +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')); + +-- name: JobGetAvailableIDs :many +SELECT id +FROM /* TEMPLATE: schema */river_job +WHERE + priority >= 0 + AND queue = sqlc.arg('queue') + AND scheduled_at <= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND state = 'available' + -- The first comparison permits use of the kind index; the binary + -- comparison preserves exact matching under PAD SPACE collations. + AND (sqlc.arg('kind_filter') = 0 OR ( + kind IN (sqlc.slice('kind')) + AND /* TEMPLATE_BEGIN: kind_column */ kind /* TEMPLATE_END */ IN (sqlc.slice('kind')) + )) +ORDER BY priority ASC, scheduled_at ASC, id ASC +LIMIT ? +FOR UPDATE SKIP LOCKED; + +-- name: JobGetAvailableUpdate :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = attempt + 1, + attempted_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + attempted_by = JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(sqlc.arg('max_attempted_by') AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(sqlc.arg('max_attempted_by') AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(sqlc.arg('max_attempted_by') AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(sqlc.arg('attempted_by') AS CHAR) + ), + state = 'running' +WHERE id IN (sqlc.slice('id')); + +-- name: JobGetByIDManyOrdered :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY priority ASC, scheduled_at ASC, id ASC; + +-- name: JobGetCancelRequested :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) + AND state = 'running' + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') +ORDER BY id; + +-- name: JobGetStuck :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE state = 'running' + AND id > sqlc.arg('after_id') + AND attempted_at < sqlc.arg('stuck_horizon') +ORDER BY id +LIMIT ?; + +-- name: JobInsertFast :execresult +INSERT INTO /* TEMPLATE: schema */river_job( + id, + args, + created_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + sqlc.narg('id'), + sqlc.arg('args'), + COALESCE(sqlc.narg('created_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('kind'), + sqlc.arg('max_attempts'), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('priority'), + sqlc.arg('queue'), + COALESCE(sqlc.narg('scheduled_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('state'), + CAST(sqlc.arg('tags') AS JSON), + sqlc.narg('unique_key'), + sqlc.narg('unique_states') +) /* TEMPLATE_BEGIN: on_duplicate_key */ ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) /* TEMPLATE_END */; + +-- name: JobInsertFullExec :execlastid +INSERT INTO /* TEMPLATE: schema */river_job( + args, + attempt, + attempted_at, + attempted_by, + created_at, + errors, + finalized_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + sqlc.arg('args'), + sqlc.arg('attempt'), + sqlc.narg('attempted_at'), + CAST(sqlc.narg('attempted_by') AS JSON), + COALESCE(sqlc.narg('created_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(sqlc.narg('errors') AS JSON), + sqlc.narg('finalized_at'), + sqlc.arg('kind'), + sqlc.arg('max_attempts'), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('priority'), + sqlc.arg('queue'), + COALESCE(sqlc.narg('scheduled_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('state'), + CAST(sqlc.arg('tags') AS JSON), + sqlc.narg('unique_key'), + sqlc.narg('unique_states') +); + +-- name: JobKindList :many +SELECT DISTINCT kind +FROM /* TEMPLATE: schema */river_job +WHERE (sqlc.arg('match') = '' OR LOWER(kind) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(sqlc.arg('match')), '%')) + AND (sqlc.arg('after') = '' OR kind > sqlc.arg('after')) + AND kind NOT IN (sqlc.slice('exclude')) +ORDER BY kind ASC +LIMIT ?; + +-- name: JobList :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ?; + +-- name: JobRescue :exec +UPDATE /* TEMPLATE: schema */river_job +SET + errors = JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(sqlc.arg('error') AS JSON)), + finalized_at = sqlc.narg('finalized_at'), + scheduled_at = sqlc.arg('scheduled_at'), + metadata = JSON_SET( + metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + state = sqlc.arg('state') +WHERE id = sqlc.arg('id') + AND state = 'running' + AND attempted_at < sqlc.arg('stuck_horizon'); + +-- name: JobRetryExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = 'available', + max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END, + finalized_at = NULL, + scheduled_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +WHERE id = sqlc.arg('id') + AND state != 'running' + AND ( + state <> 'available' + OR scheduled_at > COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ); + +-- name: JobSchedule :many +WITH eligible AS ( + SELECT river_job.id, river_job.unique_key, river_job.unique_states, river_job.priority, river_job.scheduled_at, + CASE + WHEN river_job.unique_key IS NOT NULL AND river_job.unique_states IS NOT NULL THEN + ROW_NUMBER() OVER (PARTITION BY river_job.unique_key ORDER BY river_job.priority, river_job.scheduled_at, river_job.id) + ELSE NULL + END AS row_num + FROM /* TEMPLATE: schema */river_job + WHERE + river_job.state IN ('retryable', 'scheduled') + AND river_job.scheduled_at <= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ORDER BY + river_job.priority, + river_job.scheduled_at, + river_job.id + LIMIT ? + FOR UPDATE SKIP LOCKED +), +unique_conflicts AS ( + SELECT DISTINCT eligible.unique_key + FROM /* TEMPLATE: schema */river_job + JOIN eligible + ON river_job.unique_key = eligible.unique_key + AND river_job.id != eligible.id + WHERE + river_job.unique_key IS NOT NULL + AND river_job.unique_states IS NOT NULL + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1 +) +SELECT eligible.id, + CASE + WHEN eligible.unique_key IS NULL OR eligible.unique_states IS NULL THEN FALSE + WHEN uc.unique_key IS NOT NULL THEN TRUE + WHEN eligible.row_num > 1 THEN TRUE + ELSE FALSE + END AS conflict_discarded +FROM eligible +LEFT JOIN unique_conflicts uc ON eligible.unique_key = uc.unique_key +ORDER BY eligible.priority, eligible.scheduled_at, eligible.id; + +-- name: JobScheduleSetAvailableExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET state = 'available' +WHERE id IN (sqlc.slice('id')); + +-- name: JobScheduleSetDiscardedExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET metadata = JSON_MERGE_PATCH(metadata, '{"unique_key_conflict": "scheduler_discarded"}'), + finalized_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + state = 'discarded' +WHERE id IN (sqlc.slice('id')); + +-- name: JobSetStateIfRunningExec :exec +-- MySQL evaluates assignments left-to-right. Keep state after the fields whose +-- updates depend on it, and metadata last so cancellation checks use the old metadata. +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE + WHEN state = 'running' + AND ((CAST(sqlc.arg('state') AS CHAR) <> 'available' AND sqlc.arg('state') <> 'retryable' AND sqlc.arg('state') <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(sqlc.arg('attempt_do_update') AS SIGNED) + THEN sqlc.arg('attempt') ELSE attempt END, + errors = CASE + WHEN state = 'running' AND CAST(sqlc.arg('errors_do_update') AS SIGNED) + THEN JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(sqlc.arg('error') AS JSON)) + ELSE errors END, + finalized_at = CASE + WHEN state = 'running' AND ((sqlc.arg('state') = 'available' OR sqlc.arg('state') = 'retryable' OR sqlc.arg('state') = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN state = 'running' AND CAST(sqlc.arg('finalized_at_do_update') AS SIGNED) + THEN sqlc.narg('finalized_at') ELSE finalized_at END, + scheduled_at = CASE + WHEN state = 'running' + AND ((sqlc.arg('state') <> 'available' AND sqlc.arg('state') <> 'retryable' AND sqlc.arg('state') <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(sqlc.arg('scheduled_at_do_update') AS SIGNED) + THEN sqlc.arg('scheduled_at') ELSE scheduled_at END, + state = CASE + WHEN state = 'running' AND ((sqlc.arg('state') = 'available' OR sqlc.arg('state') = 'retryable' OR sqlc.arg('state') = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN 'cancelled' + WHEN state = 'running' THEN sqlc.arg('state') ELSE state END, + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_merge') AS SIGNED) + THEN CAST(sqlc.arg('metadata_updates') AS JSON) ELSE metadata END +WHERE id = sqlc.arg('id'); + +-- name: JobUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_merge') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END +WHERE id = sqlc.arg('id'); + +-- name: JobUpdateFullExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE WHEN CAST(sqlc.arg('attempt_do_update') AS SIGNED) THEN sqlc.arg('attempt') ELSE attempt END, + attempted_at = CASE WHEN CAST(sqlc.arg('attempted_at_do_update') AS SIGNED) THEN sqlc.narg('attempted_at') ELSE attempted_at END, + attempted_by = CASE WHEN CAST(sqlc.arg('attempted_by_do_update') AS SIGNED) THEN CAST(sqlc.arg('attempted_by') AS JSON) ELSE attempted_by END, + errors = CASE WHEN CAST(sqlc.arg('errors_do_update') AS SIGNED) THEN CAST(sqlc.arg('errors') AS JSON) ELSE errors END, + finalized_at = CASE WHEN CAST(sqlc.arg('finalized_at_do_update') AS SIGNED) THEN sqlc.narg('finalized_at') ELSE finalized_at END, + max_attempts = CASE WHEN CAST(sqlc.arg('max_attempts_do_update') AS SIGNED) THEN sqlc.arg('max_attempts') ELSE max_attempts END, + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_update') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END, + state = CASE WHEN CAST(sqlc.arg('state_do_update') AS SIGNED) THEN sqlc.arg('state') ELSE state END +WHERE id = sqlc.arg('id'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go new file mode 100644 index 000000000..16ad33a62 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go @@ -0,0 +1,1642 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_job.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" + "time" +) + +const jobCancelExec = `-- name: JobCancelExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END, + finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) END, + metadata = JSON_SET(metadata, '$.cancel_attempted_at', CAST(? AS CHAR)) +WHERE id = ? + AND state NOT IN ('cancelled', 'completed', 'discarded') + AND finalized_at IS NULL +` + +type JobCancelExecParams struct { + Now sql.NullTime + CancelAttemptedAt interface{} + ID int64 +} + +func (q *Queries) JobCancelExec(ctx context.Context, db DBTX, arg *JobCancelExecParams) (sql.Result, error) { + return db.ExecContext(ctx, jobCancelExec, arg.Now, arg.CancelAttemptedAt, arg.ID) +} + +const jobClearUniqueNonce = `-- name: JobClearUniqueNonce :exec +UPDATE /* TEMPLATE: schema */river_job FORCE INDEX (PRIMARY) +SET metadata = JSON_REMOVE(metadata, '$."river:unique_nonce"') +WHERE id IN (/*SLICE:id*/?) +` + +// Avoid a table scan locking other sessions' uncommitted inserts. +func (q *Queries) JobClearUniqueNonce(ctx context.Context, db DBTX, id []int64) error { + query := jobClearUniqueNonce + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobCountByAllStates = `-- name: JobCountByAllStates :many +SELECT state, count(*) AS count +FROM /* TEMPLATE: schema */river_job +GROUP BY state +` + +type JobCountByAllStatesRow struct { + State string + Count int64 +} + +func (q *Queries) JobCountByAllStates(ctx context.Context, db DBTX) ([]*JobCountByAllStatesRow, error) { + rows, err := db.QueryContext(ctx, jobCountByAllStates) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobCountByAllStatesRow + for rows.Next() { + var i JobCountByAllStatesRow + if err := rows.Scan(&i.State, &i.Count); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobCountByQueueAndState = `-- name: JobCountByQueueAndState :many +SELECT + queue, + COUNT(CASE WHEN state = 'available' THEN 1 END) AS count_available, + COUNT(CASE WHEN state = 'running' THEN 1 END) AS count_running +FROM /* TEMPLATE: schema */river_job +WHERE queue IN (/*SLICE:queue_names*/?) + AND state IN ('available', 'running') +GROUP BY queue +ORDER BY queue +` + +type JobCountByQueueAndStateRow struct { + Queue string + CountAvailable int64 + CountRunning int64 +} + +func (q *Queries) JobCountByQueueAndState(ctx context.Context, db DBTX, queueNames []string) ([]*JobCountByQueueAndStateRow, error) { + query := jobCountByQueueAndState + var queryParams []interface{} + if len(queueNames) > 0 { + for _, v := range queueNames { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queue_names*/?", strings.Repeat(",?", len(queueNames))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queue_names*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobCountByQueueAndStateRow + for rows.Next() { + var i JobCountByQueueAndStateRow + if err := rows.Scan(&i.Queue, &i.CountAvailable, &i.CountRunning); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobCountByState = `-- name: JobCountByState :one +SELECT count(*) AS count +FROM /* TEMPLATE: schema */river_job +WHERE state = ? +` + +func (q *Queries) JobCountByState(ctx context.Context, db DBTX, state string) (int64, error) { + row := db.QueryRowContext(ctx, jobCountByState, state) + var count int64 + err := row.Scan(&count) + return count, err +} + +const jobDeleteBeforeSelect = `-- name: JobDeleteBeforeSelect :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE ( + ( + CAST(? AS UNSIGNED) + AND state = 'cancelled' + AND finalized_at < ? + ) OR ( + CAST(? AS UNSIGNED) + AND state = 'completed' + AND finalized_at < ? + ) OR ( + CAST(? AS UNSIGNED) + AND state = 'discarded' + AND finalized_at < ? + ) +) +AND ( + CAST(? AS SIGNED) + OR queue NOT IN (/*SLICE:queues_excluded*/?) +) +AND ( + CAST(? AS SIGNED) + OR queue IN (/*SLICE:queues_included*/?) +) +ORDER BY id +LIMIT ? +FOR UPDATE SKIP LOCKED +` + +type JobDeleteBeforeSelectParams struct { + CancelledDoDelete int64 + CancelledFinalizedAtHorizon sql.NullTime + CompletedDoDelete int64 + CompletedFinalizedAtHorizon sql.NullTime + DiscardedDoDelete int64 + DiscardedFinalizedAtHorizon sql.NullTime + QueuesExcludedEmpty int64 + QueuesExcluded []string + QueuesIncludedEmpty int64 + QueuesIncluded []string + Limit int32 +} + +func (q *Queries) JobDeleteBeforeSelect(ctx context.Context, db DBTX, arg *JobDeleteBeforeSelectParams) ([]int64, error) { + query := jobDeleteBeforeSelect + var queryParams []interface{} + queryParams = append(queryParams, arg.CancelledDoDelete) + queryParams = append(queryParams, arg.CancelledFinalizedAtHorizon) + queryParams = append(queryParams, arg.CompletedDoDelete) + queryParams = append(queryParams, arg.CompletedFinalizedAtHorizon) + queryParams = append(queryParams, arg.DiscardedDoDelete) + queryParams = append(queryParams, arg.DiscardedFinalizedAtHorizon) + queryParams = append(queryParams, arg.QueuesExcludedEmpty) + if len(arg.QueuesExcluded) > 0 { + for _, v := range arg.QueuesExcluded { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queues_excluded*/?", strings.Repeat(",?", len(arg.QueuesExcluded))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queues_excluded*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.QueuesIncludedEmpty) + if len(arg.QueuesIncluded) > 0 { + for _, v := range arg.QueuesIncluded { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queues_included*/?", strings.Repeat(",?", len(arg.QueuesIncluded))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queues_included*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobDeleteExec = `-- name: JobDeleteExec :execresult +DELETE FROM /* TEMPLATE: schema */river_job +WHERE id = ? + AND river_job.state != 'running' +` + +func (q *Queries) JobDeleteExec(ctx context.Context, db DBTX, id int64) (sql.Result, error) { + return db.ExecContext(ctx, jobDeleteExec, id) +} + +const jobDeleteManyExec = `-- name: JobDeleteManyExec :exec +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +` + +// Keep large batches on primary-key ranges so a scan cannot lock skipped jobs. +func (q *Queries) JobDeleteManyExec(ctx context.Context, db DBTX, id []int64) error { + query := jobDeleteManyExec + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobDeleteManySelect = `-- name: JobDeleteManySelect :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + AND state != 'running' +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +FOR UPDATE +` + +func (q *Queries) JobDeleteManySelect(ctx context.Context, db DBTX, limit int32) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobDeleteManySelect, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetAvailableIDs = `-- name: JobGetAvailableIDs :many +SELECT id +FROM /* TEMPLATE: schema */river_job +WHERE + priority >= 0 + AND queue = ? + AND scheduled_at <= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND state = 'available' + -- The first comparison permits use of the kind index; the binary + -- comparison preserves exact matching under PAD SPACE collations. + AND (? = 0 OR ( + kind IN (/*SLICE:kind*/?) + AND /* TEMPLATE_BEGIN: kind_column */ kind /* TEMPLATE_END */ IN (/*SLICE:kind*/?) + )) +ORDER BY priority ASC, scheduled_at ASC, id ASC +LIMIT ? +FOR UPDATE SKIP LOCKED +` + +type JobGetAvailableIDsParams struct { + Queue string + Now sql.NullTime + KindFilter interface{} + Kind []string + Limit int32 +} + +func (q *Queries) JobGetAvailableIDs(ctx context.Context, db DBTX, arg *JobGetAvailableIDsParams) ([]int64, error) { + query := jobGetAvailableIDs + var queryParams []interface{} + queryParams = append(queryParams, arg.Queue) + queryParams = append(queryParams, arg.Now) + queryParams = append(queryParams, arg.KindFilter) + if len(arg.Kind) > 0 { + for _, v := range arg.Kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(arg.Kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + if len(arg.Kind) > 0 { + for _, v := range arg.Kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(arg.Kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetAvailableUpdate = `-- name: JobGetAvailableUpdate :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = attempt + 1, + attempted_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + attempted_by = JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(? AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(? AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(? AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(? AS CHAR) + ), + state = 'running' +WHERE id IN (/*SLICE:id*/?) +` + +type JobGetAvailableUpdateParams struct { + Now sql.NullTime + MaxAttemptedBy int64 + AttemptedBy interface{} + ID []int64 +} + +func (q *Queries) JobGetAvailableUpdate(ctx context.Context, db DBTX, arg *JobGetAvailableUpdateParams) error { + query := jobGetAvailableUpdate + var queryParams []interface{} + queryParams = append(queryParams, arg.Now) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.AttemptedBy) + if len(arg.ID) > 0 { + for _, v := range arg.ID { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(arg.ID))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobGetByID = `-- name: JobGetByID :one +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id = ? +LIMIT 1 +` + +func (q *Queries) JobGetByID(ctx context.Context, db DBTX, id int64) (*RiverJob, error) { + row := db.QueryRowContext(ctx, jobGetByID, id) + var i RiverJob + err := row.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ) + return &i, err +} + +const jobGetByIDForUpdate = `-- name: JobGetByIDForUpdate :one +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id = ? +LIMIT 1 +FOR UPDATE +` + +func (q *Queries) JobGetByIDForUpdate(ctx context.Context, db DBTX, id int64) (*RiverJob, error) { + row := db.QueryRowContext(ctx, jobGetByIDForUpdate, id) + var i RiverJob + err := row.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ) + return &i, err +} + +const jobGetByIDMany = `-- name: JobGetByIDMany :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +` + +func (q *Queries) JobGetByIDMany(ctx context.Context, db DBTX, id []int64) ([]*RiverJob, error) { + query := jobGetByIDMany + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByIDManyForUpdate = `-- name: JobGetByIDManyForUpdate :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +FOR UPDATE +` + +// Read current versions after writes, even in a pre-existing REPEATABLE READ transaction. +func (q *Queries) JobGetByIDManyForUpdate(ctx context.Context, db DBTX, id []int64) ([]*RiverJob, error) { + query := jobGetByIDManyForUpdate + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByIDManyOrdered = `-- name: JobGetByIDManyOrdered :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY priority ASC, scheduled_at ASC, id ASC +` + +func (q *Queries) JobGetByIDManyOrdered(ctx context.Context, db DBTX, id []int64) ([]*RiverJob, error) { + query := jobGetByIDManyOrdered + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByInsertNonce = `-- name: JobGetByInsertNonce :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id >= ? + AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$."river:unique_nonce"')) LIKE ? +ORDER BY id +` + +type JobGetByInsertNonceParams struct { + FirstID int64 + NoncePrefix []byte +} + +// Own writes are visible even in an older REPEATABLE READ snapshot. Do not use +// a locking read: the range can contain interleaved inserts from other sessions. +func (q *Queries) JobGetByInsertNonce(ctx context.Context, db DBTX, arg *JobGetByInsertNonceParams) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobGetByInsertNonce, arg.FirstID, arg.NoncePrefix) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByKindMany = `-- name: JobGetByKindMany :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE kind IN (/*SLICE:kind*/?) +ORDER BY id +` + +func (q *Queries) JobGetByKindMany(ctx context.Context, db DBTX, kind []string) ([]*RiverJob, error) { + query := jobGetByKindMany + var queryParams []interface{} + if len(kind) > 0 { + for _, v := range kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByUniqueKeyManyForUpdate = `-- name: JobGetByUniqueKeyManyForUpdate :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job FORCE INDEX (river_job_unique_idx) +WHERE /* TEMPLATE_BEGIN: unique_key */ unique_key /* TEMPLATE_END */ IN (/*SLICE:unique_keys*/?) +ORDER BY id +FOR UPDATE +` + +// Match the expression in river_job_unique_idx so lookups use the unique index. +// A current read is needed for existing jobs committed after a caller's snapshot. +// Force the index to avoid locking unrelated rows if a table scan looks cheaper. +func (q *Queries) JobGetByUniqueKeyManyForUpdate(ctx context.Context, db DBTX, uniqueKeys []sql.NullString) ([]*RiverJob, error) { + query := jobGetByUniqueKeyManyForUpdate + var queryParams []interface{} + if len(uniqueKeys) > 0 { + for _, v := range uniqueKeys { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:unique_keys*/?", strings.Repeat(",?", len(uniqueKeys))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:unique_keys*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetCancelRequested = `-- name: JobGetCancelRequested :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) + AND state = 'running' + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') +ORDER BY id +` + +func (q *Queries) JobGetCancelRequested(ctx context.Context, db DBTX, id []int64) ([]int64, error) { + query := jobGetCancelRequested + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetMetadataForUpdate = `-- name: JobGetMetadataForUpdate :many +SELECT /*+ JOIN_INDEX(river_job PRIMARY) */ id, metadata +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +FOR UPDATE +` + +type JobGetMetadataForUpdateRow struct { + ID int64 + Metadata []byte +} + +func (q *Queries) JobGetMetadataForUpdate(ctx context.Context, db DBTX, id []int64) ([]*JobGetMetadataForUpdateRow, error) { + query := jobGetMetadataForUpdate + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobGetMetadataForUpdateRow + for rows.Next() { + var i JobGetMetadataForUpdateRow + if err := rows.Scan(&i.ID, &i.Metadata); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetStuck = `-- name: JobGetStuck :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE state = 'running' + AND id > ? + AND attempted_at < ? +ORDER BY id +LIMIT ? +` + +type JobGetStuckParams struct { + AfterID int64 + StuckHorizon sql.NullTime + Limit int32 +} + +func (q *Queries) JobGetStuck(ctx context.Context, db DBTX, arg *JobGetStuckParams) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobGetStuck, arg.AfterID, arg.StuckHorizon, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobInsertFast = `-- name: JobInsertFast :execresult +INSERT INTO /* TEMPLATE: schema */river_job( + id, + args, + created_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + ?, + CAST(? AS JSON), + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + CAST(? AS JSON), + ?, + ? +) /* TEMPLATE_BEGIN: on_duplicate_key */ ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) /* TEMPLATE_END */ +` + +type JobInsertFastParams struct { + ID sql.NullInt64 + Args []byte + CreatedAt interface{} + Kind string + MaxAttempts int64 + Metadata []byte + Priority int16 + Queue string + ScheduledAt interface{} + State string + Tags []byte + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +func (q *Queries) JobInsertFast(ctx context.Context, db DBTX, arg *JobInsertFastParams) (sql.Result, error) { + return db.ExecContext(ctx, jobInsertFast, + arg.ID, + arg.Args, + arg.CreatedAt, + arg.Kind, + arg.MaxAttempts, + arg.Metadata, + arg.Priority, + arg.Queue, + arg.ScheduledAt, + arg.State, + arg.Tags, + arg.UniqueKey, + arg.UniqueStates, + ) +} + +const jobInsertFullExec = `-- name: JobInsertFullExec :execlastid +INSERT INTO /* TEMPLATE: schema */river_job( + args, + attempt, + attempted_at, + attempted_by, + created_at, + errors, + finalized_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + ?, + ?, + ?, + CAST(? AS JSON), + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(? AS JSON), + ?, + ?, + ?, + CAST(? AS JSON), + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + CAST(? AS JSON), + ?, + ? +) +` + +type JobInsertFullExecParams struct { + Args []byte + Attempt int64 + AttemptedAt sql.NullTime + AttemptedBy []byte + CreatedAt interface{} + Errors []byte + FinalizedAt sql.NullTime + Kind string + MaxAttempts int64 + Metadata []byte + Priority int16 + Queue string + ScheduledAt interface{} + State string + Tags []byte + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +func (q *Queries) JobInsertFullExec(ctx context.Context, db DBTX, arg *JobInsertFullExecParams) (int64, error) { + result, err := db.ExecContext(ctx, jobInsertFullExec, + arg.Args, + arg.Attempt, + arg.AttemptedAt, + arg.AttemptedBy, + arg.CreatedAt, + arg.Errors, + arg.FinalizedAt, + arg.Kind, + arg.MaxAttempts, + arg.Metadata, + arg.Priority, + arg.Queue, + arg.ScheduledAt, + arg.State, + arg.Tags, + arg.UniqueKey, + arg.UniqueStates, + ) + if err != nil { + return 0, err + } + return result.LastInsertId() +} + +const jobKindList = `-- name: JobKindList :many +SELECT DISTINCT kind +FROM /* TEMPLATE: schema */river_job +WHERE (? = '' OR LOWER(kind) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(?), '%')) + AND (? = '' OR kind > ?) + AND kind NOT IN (/*SLICE:exclude*/?) +ORDER BY kind ASC +LIMIT ? +` + +type JobKindListParams struct { + Match string + After string + Exclude []string + Limit int32 +} + +func (q *Queries) JobKindList(ctx context.Context, db DBTX, arg *JobKindListParams) ([]string, error) { + query := jobKindList + var queryParams []interface{} + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.After) + queryParams = append(queryParams, arg.After) + if len(arg.Exclude) > 0 { + for _, v := range arg.Exclude { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:exclude*/?", strings.Repeat(",?", len(arg.Exclude))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:exclude*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var kind string + if err := rows.Scan(&kind); err != nil { + return nil, err + } + items = append(items, kind) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobList = `-- name: JobList :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +` + +func (q *Queries) JobList(ctx context.Context, db DBTX, limit int32) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobList, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobRescue = `-- name: JobRescue :exec +UPDATE /* TEMPLATE: schema */river_job +SET + errors = JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(? AS JSON)), + finalized_at = ?, + scheduled_at = ?, + metadata = JSON_SET( + metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + state = ? +WHERE id = ? + AND state = 'running' + AND attempted_at < ? +` + +type JobRescueParams struct { + Error []byte + FinalizedAt sql.NullTime + ScheduledAt time.Time + State string + ID int64 + StuckHorizon sql.NullTime +} + +func (q *Queries) JobRescue(ctx context.Context, db DBTX, arg *JobRescueParams) error { + _, err := db.ExecContext(ctx, jobRescue, + arg.Error, + arg.FinalizedAt, + arg.ScheduledAt, + arg.State, + arg.ID, + arg.StuckHorizon, + ) + return err +} + +const jobRetryExec = `-- name: JobRetryExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = 'available', + max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END, + finalized_at = NULL, + scheduled_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +WHERE id = ? + AND state != 'running' + AND ( + state <> 'available' + OR scheduled_at > COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ) +` + +type JobRetryExecParams struct { + Now sql.NullTime + ID int64 +} + +func (q *Queries) JobRetryExec(ctx context.Context, db DBTX, arg *JobRetryExecParams) (sql.Result, error) { + return db.ExecContext(ctx, jobRetryExec, arg.Now, arg.ID, arg.Now) +} + +const jobSchedule = `-- name: JobSchedule :many +WITH eligible AS ( + SELECT river_job.id, river_job.unique_key, river_job.unique_states, river_job.priority, river_job.scheduled_at, + CASE + WHEN river_job.unique_key IS NOT NULL AND river_job.unique_states IS NOT NULL THEN + ROW_NUMBER() OVER (PARTITION BY river_job.unique_key ORDER BY river_job.priority, river_job.scheduled_at, river_job.id) + ELSE NULL + END AS row_num + FROM /* TEMPLATE: schema */river_job + WHERE + river_job.state IN ('retryable', 'scheduled') + AND river_job.scheduled_at <= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ORDER BY + river_job.priority, + river_job.scheduled_at, + river_job.id + LIMIT ? + FOR UPDATE SKIP LOCKED +), +unique_conflicts AS ( + SELECT DISTINCT eligible.unique_key + FROM /* TEMPLATE: schema */river_job + JOIN eligible + ON river_job.unique_key = eligible.unique_key + AND river_job.id != eligible.id + WHERE + river_job.unique_key IS NOT NULL + AND river_job.unique_states IS NOT NULL + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1 +) +SELECT eligible.id, + CASE + WHEN eligible.unique_key IS NULL OR eligible.unique_states IS NULL THEN FALSE + WHEN uc.unique_key IS NOT NULL THEN TRUE + WHEN eligible.row_num > 1 THEN TRUE + ELSE FALSE + END AS conflict_discarded +FROM eligible +LEFT JOIN unique_conflicts uc ON eligible.unique_key = uc.unique_key +ORDER BY eligible.priority, eligible.scheduled_at, eligible.id +` + +type JobScheduleParams struct { + Now sql.NullTime + Limit int32 +} + +type JobScheduleRow struct { + ID int64 + ConflictDiscarded int64 +} + +func (q *Queries) JobSchedule(ctx context.Context, db DBTX, arg *JobScheduleParams) ([]*JobScheduleRow, error) { + rows, err := db.QueryContext(ctx, jobSchedule, arg.Now, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobScheduleRow + for rows.Next() { + var i JobScheduleRow + if err := rows.Scan(&i.ID, &i.ConflictDiscarded); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobScheduleSetAvailableExec = `-- name: JobScheduleSetAvailableExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET state = 'available' +WHERE id IN (/*SLICE:id*/?) +` + +func (q *Queries) JobScheduleSetAvailableExec(ctx context.Context, db DBTX, id []int64) error { + query := jobScheduleSetAvailableExec + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobScheduleSetDiscardedExec = `-- name: JobScheduleSetDiscardedExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET metadata = JSON_MERGE_PATCH(metadata, '{"unique_key_conflict": "scheduler_discarded"}'), + finalized_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + state = 'discarded' +WHERE id IN (/*SLICE:id*/?) +` + +type JobScheduleSetDiscardedExecParams struct { + Now sql.NullTime + ID []int64 +} + +func (q *Queries) JobScheduleSetDiscardedExec(ctx context.Context, db DBTX, arg *JobScheduleSetDiscardedExecParams) error { + query := jobScheduleSetDiscardedExec + var queryParams []interface{} + queryParams = append(queryParams, arg.Now) + if len(arg.ID) > 0 { + for _, v := range arg.ID { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(arg.ID))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobSetStateIfRunningExec = `-- name: JobSetStateIfRunningExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE + WHEN state = 'running' + AND ((CAST(? AS CHAR) <> 'available' AND ? <> 'retryable' AND ? <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(? AS SIGNED) + THEN ? ELSE attempt END, + errors = CASE + WHEN state = 'running' AND CAST(? AS SIGNED) + THEN JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(? AS JSON)) + ELSE errors END, + finalized_at = CASE + WHEN state = 'running' AND ((? = 'available' OR ? = 'retryable' OR ? = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN state = 'running' AND CAST(? AS SIGNED) + THEN ? ELSE finalized_at END, + scheduled_at = CASE + WHEN state = 'running' + AND ((? <> 'available' AND ? <> 'retryable' AND ? <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(? AS SIGNED) + THEN ? ELSE scheduled_at END, + state = CASE + WHEN state = 'running' AND ((? = 'available' OR ? = 'retryable' OR ? = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN 'cancelled' + WHEN state = 'running' THEN ? ELSE state END, + metadata = CASE WHEN CAST(? AS SIGNED) + THEN CAST(? AS JSON) ELSE metadata END +WHERE id = ? +` + +type JobSetStateIfRunningExecParams struct { + State string + AttemptDoUpdate int64 + Attempt int64 + ErrorsDoUpdate int64 + Error []byte + Now sql.NullTime + FinalizedAtDoUpdate int64 + FinalizedAt sql.NullTime + ScheduledAtDoUpdate int64 + ScheduledAt time.Time + MetadataDoMerge int64 + MetadataUpdates []byte + ID int64 +} + +// MySQL evaluates assignments left-to-right. Keep state after the fields whose +// updates depend on it, and metadata last so cancellation checks use the old metadata. +func (q *Queries) JobSetStateIfRunningExec(ctx context.Context, db DBTX, arg *JobSetStateIfRunningExecParams) error { + _, err := db.ExecContext(ctx, jobSetStateIfRunningExec, + arg.State, + arg.State, + arg.State, + arg.AttemptDoUpdate, + arg.Attempt, + arg.ErrorsDoUpdate, + arg.Error, + arg.State, + arg.State, + arg.State, + arg.Now, + arg.FinalizedAtDoUpdate, + arg.FinalizedAt, + arg.State, + arg.State, + arg.State, + arg.ScheduledAtDoUpdate, + arg.ScheduledAt, + arg.State, + arg.State, + arg.State, + arg.State, + arg.MetadataDoMerge, + arg.MetadataUpdates, + arg.ID, + ) + return err +} + +const jobUpdateExec = `-- name: JobUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END +WHERE id = ? +` + +type JobUpdateExecParams struct { + MetadataDoMerge int64 + Metadata []byte + ID int64 +} + +func (q *Queries) JobUpdateExec(ctx context.Context, db DBTX, arg *JobUpdateExecParams) error { + _, err := db.ExecContext(ctx, jobUpdateExec, arg.MetadataDoMerge, arg.Metadata, arg.ID) + return err +} + +const jobUpdateFullExec = `-- name: JobUpdateFullExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE attempt END, + attempted_at = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE attempted_at END, + attempted_by = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE attempted_by END, + errors = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE errors END, + finalized_at = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE finalized_at END, + max_attempts = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE max_attempts END, + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END, + state = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE state END +WHERE id = ? +` + +type JobUpdateFullExecParams struct { + AttemptDoUpdate int64 + Attempt int64 + AttemptedAtDoUpdate int64 + AttemptedAt sql.NullTime + AttemptedByDoUpdate int64 + AttemptedBy []byte + ErrorsDoUpdate int64 + Errors []byte + FinalizedAtDoUpdate int64 + FinalizedAt sql.NullTime + MaxAttemptsDoUpdate int64 + MaxAttempts int64 + MetadataDoUpdate int64 + Metadata []byte + StateDoUpdate int64 + State string + ID int64 +} + +func (q *Queries) JobUpdateFullExec(ctx context.Context, db DBTX, arg *JobUpdateFullExecParams) error { + _, err := db.ExecContext(ctx, jobUpdateFullExec, + arg.AttemptDoUpdate, + arg.Attempt, + arg.AttemptedAtDoUpdate, + arg.AttemptedAt, + arg.AttemptedByDoUpdate, + arg.AttemptedBy, + arg.ErrorsDoUpdate, + arg.Errors, + arg.FinalizedAtDoUpdate, + arg.FinalizedAt, + arg.MaxAttemptsDoUpdate, + arg.MaxAttempts, + arg.MetadataDoUpdate, + arg.Metadata, + arg.StateDoUpdate, + arg.State, + arg.ID, + ) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql new file mode 100644 index 000000000..cc419b1b1 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql @@ -0,0 +1,60 @@ +CREATE TABLE river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default' PRIMARY KEY +); + +-- name: LeaderAttemptElectExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + leader_id, + elected_at, + expires_at +) VALUES ( + sqlc.arg('leader_id'), + COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +); + +-- name: LeaderAttemptReelectExec :exec +UPDATE /* TEMPLATE: schema */river_leader +SET expires_at = TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +WHERE + elected_at = sqlc.arg('elected_at') + AND expires_at >= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = sqlc.arg('leader_id'); + +-- name: LeaderDeleteExpired :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE expires_at < COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')); + +-- name: LeaderGetElectedLeader :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader; + +-- name: LeaderGetReelected :one +-- Matched renewals can leave expires_at unchanged, which MySQL reports as zero +-- affected rows. Check the current term instead of using that row count. +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +WHERE elected_at = sqlc.arg('elected_at') + AND expires_at >= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = sqlc.arg('leader_id') +FOR UPDATE; + +-- name: LeaderInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + elected_at, + expires_at, + leader_id +) VALUES ( + COALESCE(sqlc.narg('elected_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + COALESCE(sqlc.narg('expires_at'), TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')))), + sqlc.arg('leader_id') +); + +-- name: LeaderResign :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE + elected_at = sqlc.arg('elected_at') + AND leader_id = sqlc.arg('leader_id'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go new file mode 100644 index 000000000..c620a9f4a --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go @@ -0,0 +1,178 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_leader.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "time" +) + +const leaderAttemptElectExec = `-- name: LeaderAttemptElectExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + leader_id, + elected_at, + expires_at +) VALUES ( + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +) +` + +type LeaderAttemptElectExecParams struct { + LeaderID string + Now interface{} + TTL int64 +} + +func (q *Queries) LeaderAttemptElectExec(ctx context.Context, db DBTX, arg *LeaderAttemptElectExecParams) error { + _, err := db.ExecContext(ctx, leaderAttemptElectExec, + arg.LeaderID, + arg.Now, + arg.TTL, + arg.Now, + ) + return err +} + +const leaderAttemptReelectExec = `-- name: LeaderAttemptReelectExec :exec +UPDATE /* TEMPLATE: schema */river_leader +SET expires_at = TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +WHERE + elected_at = ? + AND expires_at >= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = ? +` + +type LeaderAttemptReelectExecParams struct { + TTL int64 + Now sql.NullTime + ElectedAt time.Time + LeaderID string +} + +func (q *Queries) LeaderAttemptReelectExec(ctx context.Context, db DBTX, arg *LeaderAttemptReelectExecParams) error { + _, err := db.ExecContext(ctx, leaderAttemptReelectExec, + arg.TTL, + arg.Now, + arg.ElectedAt, + arg.Now, + arg.LeaderID, + ) + return err +} + +const leaderDeleteExpired = `-- name: LeaderDeleteExpired :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE expires_at < COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +` + +func (q *Queries) LeaderDeleteExpired(ctx context.Context, db DBTX, now sql.NullTime) (int64, error) { + result, err := db.ExecContext(ctx, leaderDeleteExpired, now) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const leaderGetElectedLeader = `-- name: LeaderGetElectedLeader :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +` + +func (q *Queries) LeaderGetElectedLeader(ctx context.Context, db DBTX) (*RiverLeader, error) { + row := db.QueryRowContext(ctx, leaderGetElectedLeader) + var i RiverLeader + err := row.Scan( + &i.ElectedAt, + &i.ExpiresAt, + &i.LeaderID, + &i.Name, + ) + return &i, err +} + +const leaderGetReelected = `-- name: LeaderGetReelected :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +WHERE elected_at = ? + AND expires_at >= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = ? +FOR UPDATE +` + +type LeaderGetReelectedParams struct { + ElectedAt time.Time + Now sql.NullTime + LeaderID string +} + +// Matched renewals can leave expires_at unchanged, which MySQL reports as zero +// affected rows. Check the current term instead of using that row count. +func (q *Queries) LeaderGetReelected(ctx context.Context, db DBTX, arg *LeaderGetReelectedParams) (*RiverLeader, error) { + row := db.QueryRowContext(ctx, leaderGetReelected, arg.ElectedAt, arg.Now, arg.LeaderID) + var i RiverLeader + err := row.Scan( + &i.ElectedAt, + &i.ExpiresAt, + &i.LeaderID, + &i.Name, + ) + return &i, err +} + +const leaderInsertExec = `-- name: LeaderInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + elected_at, + expires_at, + leader_id +) VALUES ( + COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + COALESCE(?, TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')))), + ? +) +` + +type LeaderInsertExecParams struct { + ElectedAt interface{} + Now interface{} + ExpiresAt interface{} + TTL int64 + LeaderID string +} + +func (q *Queries) LeaderInsertExec(ctx context.Context, db DBTX, arg *LeaderInsertExecParams) error { + _, err := db.ExecContext(ctx, leaderInsertExec, + arg.ElectedAt, + arg.Now, + arg.ExpiresAt, + arg.TTL, + arg.Now, + arg.LeaderID, + ) + return err +} + +const leaderResign = `-- name: LeaderResign :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE + elected_at = ? + AND leader_id = ? +` + +type LeaderResignParams struct { + ElectedAt time.Time + LeaderID string +} + +func (q *Queries) LeaderResign(ctx context.Context, db DBTX, arg *LeaderResignParams) (int64, error) { + result, err := db.ExecContext(ctx, leaderResign, arg.ElectedAt, arg.LeaderID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql new file mode 100644 index 000000000..30849a132 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql @@ -0,0 +1,80 @@ +CREATE TABLE river_migration ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + PRIMARY KEY (line, version) +); + +-- name: RiverMigrationDeleteAssumingMainMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteAssumingMainManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') + AND version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteByLineAndVersionManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') + AND version IN (sqlc.slice('version')); + +-- name: RiverMigrationGetAllAssumingMain :many +SELECT + created_at, + version +FROM /* TEMPLATE: schema */river_migration +ORDER BY version; + +-- name: RiverMigrationGetByLine :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') +ORDER BY version; + +-- name: RiverMigrationInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + line, + version +) VALUES ( + sqlc.arg('line'), + sqlc.arg('version') +); + +-- name: RiverMigrationGetByLineAndVersion :one +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') AND version = sqlc.arg('version'); + +-- name: RiverMigrationGetByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') AND version IN (sqlc.slice('version')) +ORDER BY version; + +-- name: RiverMigrationInsertAssumingMainExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + version +) VALUES ( + sqlc.arg('version') +); + +-- name: RiverMigrationGetByVersion :one +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version = sqlc.arg('version'); + +-- name: RiverMigrationGetByVersionMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')) +ORDER BY version; diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go new file mode 100644 index 000000000..5238d279f --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go @@ -0,0 +1,375 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_migration.sql + +package dbsqlc + +import ( + "context" + "strings" + "time" +) + +const riverMigrationDeleteAssumingMainMany = `-- name: RiverMigrationDeleteAssumingMainMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteAssumingMainManyRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationDeleteAssumingMainMany(ctx context.Context, db DBTX, version []int64) ([]*RiverMigrationDeleteAssumingMainManyRow, error) { + query := riverMigrationDeleteAssumingMainMany + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationDeleteAssumingMainManyRow + for rows.Next() { + var i RiverMigrationDeleteAssumingMainManyRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationDeleteAssumingMainManyExec = `-- name: RiverMigrationDeleteAssumingMainManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +` + +func (q *Queries) RiverMigrationDeleteAssumingMainManyExec(ctx context.Context, db DBTX, version []int64) error { + query := riverMigrationDeleteAssumingMainManyExec + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const riverMigrationDeleteByLineAndVersionMany = `-- name: RiverMigrationDeleteByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? + AND version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteByLineAndVersionManyParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationDeleteByLineAndVersionMany(ctx context.Context, db DBTX, arg *RiverMigrationDeleteByLineAndVersionManyParams) ([]*RiverMigration, error) { + query := riverMigrationDeleteByLineAndVersionMany + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationDeleteByLineAndVersionManyExec = `-- name: RiverMigrationDeleteByLineAndVersionManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE line = ? + AND version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteByLineAndVersionManyExecParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationDeleteByLineAndVersionManyExec(ctx context.Context, db DBTX, arg *RiverMigrationDeleteByLineAndVersionManyExecParams) error { + query := riverMigrationDeleteByLineAndVersionManyExec + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const riverMigrationGetAllAssumingMain = `-- name: RiverMigrationGetAllAssumingMain :many +SELECT + created_at, + version +FROM /* TEMPLATE: schema */river_migration +ORDER BY version +` + +type RiverMigrationGetAllAssumingMainRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetAllAssumingMain(ctx context.Context, db DBTX) ([]*RiverMigrationGetAllAssumingMainRow, error) { + rows, err := db.QueryContext(ctx, riverMigrationGetAllAssumingMain) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationGetAllAssumingMainRow + for rows.Next() { + var i RiverMigrationGetAllAssumingMainRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByLine = `-- name: RiverMigrationGetByLine :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? +ORDER BY version +` + +func (q *Queries) RiverMigrationGetByLine(ctx context.Context, db DBTX, line string) ([]*RiverMigration, error) { + rows, err := db.QueryContext(ctx, riverMigrationGetByLine, line) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByLineAndVersion = `-- name: RiverMigrationGetByLineAndVersion :one +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? AND version = ? +` + +type RiverMigrationGetByLineAndVersionParams struct { + Line string + Version int64 +} + +func (q *Queries) RiverMigrationGetByLineAndVersion(ctx context.Context, db DBTX, arg *RiverMigrationGetByLineAndVersionParams) (*RiverMigration, error) { + row := db.QueryRowContext(ctx, riverMigrationGetByLineAndVersion, arg.Line, arg.Version) + var i RiverMigration + err := row.Scan(&i.Line, &i.Version, &i.CreatedAt) + return &i, err +} + +const riverMigrationGetByLineAndVersionMany = `-- name: RiverMigrationGetByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? AND version IN (/*SLICE:version*/?) +ORDER BY version +` + +type RiverMigrationGetByLineAndVersionManyParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationGetByLineAndVersionMany(ctx context.Context, db DBTX, arg *RiverMigrationGetByLineAndVersionManyParams) ([]*RiverMigration, error) { + query := riverMigrationGetByLineAndVersionMany + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByVersion = `-- name: RiverMigrationGetByVersion :one +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version = ? +` + +type RiverMigrationGetByVersionRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetByVersion(ctx context.Context, db DBTX, version int64) (*RiverMigrationGetByVersionRow, error) { + row := db.QueryRowContext(ctx, riverMigrationGetByVersion, version) + var i RiverMigrationGetByVersionRow + err := row.Scan(&i.CreatedAt, &i.Version) + return &i, err +} + +const riverMigrationGetByVersionMany = `-- name: RiverMigrationGetByVersionMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +ORDER BY version +` + +type RiverMigrationGetByVersionManyRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetByVersionMany(ctx context.Context, db DBTX, version []int64) ([]*RiverMigrationGetByVersionManyRow, error) { + query := riverMigrationGetByVersionMany + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationGetByVersionManyRow + for rows.Next() { + var i RiverMigrationGetByVersionManyRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationInsertAssumingMainExec = `-- name: RiverMigrationInsertAssumingMainExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + version +) VALUES ( + ? +) +` + +func (q *Queries) RiverMigrationInsertAssumingMainExec(ctx context.Context, db DBTX, version int64) error { + _, err := db.ExecContext(ctx, riverMigrationInsertAssumingMainExec, version) + return err +} + +const riverMigrationInsertExec = `-- name: RiverMigrationInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + line, + version +) VALUES ( + ?, + ? +) +` + +type RiverMigrationInsertExecParams struct { + Line string + Version int64 +} + +func (q *Queries) RiverMigrationInsertExec(ctx context.Context, db DBTX, arg *RiverMigrationInsertExecParams) error { + _, err := db.ExecContext(ctx, riverMigrationInsertExec, arg.Line, arg.Version) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql new file mode 100644 index 000000000..34a2645b1 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql @@ -0,0 +1,52 @@ +CREATE TABLE river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +); + +-- name: NotificationDeleteBefore :execrows +DELETE FROM /* TEMPLATE: schema */river_notification +WHERE created_at < sqlc.arg('created_at_horizon') +ORDER BY created_at, id +LIMIT ?; + +-- name: NotificationGetAfter :many +-- InnoDB allocates auto-increment IDs before commit, so IDs may commit out of +-- order. The ascending locking read waits for an earlier uncommitted row rather +-- than returning a later committed row and causing the listener to skip it. +SELECT * +FROM /* TEMPLATE: schema */river_notification +WHERE id > sqlc.arg('after') +ORDER BY id ASC +LIMIT ? +FOR SHARE; + +-- name: NotificationGetIDsForUpdate :many +-- Used to establish a listener's initial high-water mark. This intentionally +-- scans in ascending order instead of using MAX(id) or a descending LIMIT so +-- that it encounters and waits for any lower uncommitted auto-increment IDs. +SELECT id +FROM /* TEMPLATE: schema */river_notification +ORDER BY id +FOR UPDATE; + +-- name: NotificationInsert :exec +INSERT INTO /* TEMPLATE: schema */river_notification ( + payload, + topic +) VALUES ( + sqlc.arg('payload'), + sqlc.arg('topic') +); + +-- name: NotificationInsertJobCancel :exec +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +VALUES ( + JSON_OBJECT('action', 'cancel', 'job_id', CAST(sqlc.arg('job_id') AS SIGNED), 'queue', CAST(sqlc.arg('queue') AS CHAR)), + sqlc.arg('topic') +); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go new file mode 100644 index 000000000..b178e6585 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go @@ -0,0 +1,148 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_notification.sql + +package dbsqlc + +import ( + "context" + "time" +) + +const notificationDeleteBefore = `-- name: NotificationDeleteBefore :execrows +DELETE FROM /* TEMPLATE: schema */river_notification +WHERE created_at < ? +ORDER BY created_at, id +LIMIT ? +` + +type NotificationDeleteBeforeParams struct { + CreatedAtHorizon time.Time + Limit int32 +} + +func (q *Queries) NotificationDeleteBefore(ctx context.Context, db DBTX, arg *NotificationDeleteBeforeParams) (int64, error) { + result, err := db.ExecContext(ctx, notificationDeleteBefore, arg.CreatedAtHorizon, arg.Limit) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const notificationGetAfter = `-- name: NotificationGetAfter :many +SELECT id, created_at, payload, topic +FROM /* TEMPLATE: schema */river_notification +WHERE id > ? +ORDER BY id ASC +LIMIT ? +FOR SHARE +` + +type NotificationGetAfterParams struct { + After int64 + Limit int32 +} + +// InnoDB allocates auto-increment IDs before commit, so IDs may commit out of +// order. The ascending locking read waits for an earlier uncommitted row rather +// than returning a later committed row and causing the listener to skip it. +func (q *Queries) NotificationGetAfter(ctx context.Context, db DBTX, arg *NotificationGetAfterParams) ([]*RiverNotification, error) { + rows, err := db.QueryContext(ctx, notificationGetAfter, arg.After, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverNotification + for rows.Next() { + var i RiverNotification + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.Payload, + &i.Topic, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const notificationGetIDsForUpdate = `-- name: NotificationGetIDsForUpdate :many +SELECT id +FROM /* TEMPLATE: schema */river_notification +ORDER BY id +FOR UPDATE +` + +// Used to establish a listener's initial high-water mark. This intentionally +// scans in ascending order instead of using MAX(id) or a descending LIMIT so +// that it encounters and waits for any lower uncommitted auto-increment IDs. +func (q *Queries) NotificationGetIDsForUpdate(ctx context.Context, db DBTX) ([]int64, error) { + rows, err := db.QueryContext(ctx, notificationGetIDsForUpdate) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const notificationInsert = `-- name: NotificationInsert :exec +INSERT INTO /* TEMPLATE: schema */river_notification ( + payload, + topic +) VALUES ( + ?, + ? +) +` + +type NotificationInsertParams struct { + Payload string + Topic string +} + +func (q *Queries) NotificationInsert(ctx context.Context, db DBTX, arg *NotificationInsertParams) error { + _, err := db.ExecContext(ctx, notificationInsert, arg.Payload, arg.Topic) + return err +} + +const notificationInsertJobCancel = `-- name: NotificationInsertJobCancel :exec +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +VALUES ( + JSON_OBJECT('action', 'cancel', 'job_id', CAST(? AS SIGNED), 'queue', CAST(? AS CHAR)), + ? +) +` + +type NotificationInsertJobCancelParams struct { + JobID int64 + Queue interface{} + Topic string +} + +func (q *Queries) NotificationInsertJobCancel(ctx context.Context, db DBTX, arg *NotificationInsertJobCancelParams) error { + _, err := db.ExecContext(ctx, notificationInsertJobCancel, arg.JobID, arg.Queue, arg.Topic) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql new file mode 100644 index 000000000..f559bf150 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql @@ -0,0 +1,101 @@ +CREATE TABLE river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL DEFAULT (NOW(6)) +); + +-- name: QueueCreateOrSetUpdatedAtExec :exec +INSERT INTO /* TEMPLATE: schema */river_queue ( + created_at, + metadata, + name, + paused_at, + updated_at +) VALUES ( + COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('name'), + sqlc.narg('paused_at'), + COALESCE(sqlc.narg('updated_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +) ON DUPLICATE KEY UPDATE + updated_at = COALESCE(sqlc.narg('updated_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')); + +-- name: QueueGet :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = sqlc.arg('name'); + +-- name: QueueGetForUpdate :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = sqlc.arg('name') +FOR UPDATE; + +-- name: QueueDeleteExpiredSelect :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE updated_at < sqlc.arg('updated_at_horizon') +ORDER BY name ASC +LIMIT ? +FOR UPDATE; + +-- name: QueueDeleteExpiredExec :exec +DELETE FROM /* TEMPLATE: schema */river_queue +WHERE name IN (sqlc.slice('names')); + +-- name: QueueList :many +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +ORDER BY name ASC +LIMIT ?; + +-- name: QueueNameList :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE + name > sqlc.arg('after') + AND (sqlc.arg('match') = '' OR LOWER(name) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(sqlc.arg('match')), '%')) + AND name NOT IN (sqlc.slice('exclude')) +ORDER BY name ASC +LIMIT ?; + +-- MySQL evaluates SET clauses left-to-right using already-updated values, so +-- updated_at must be set BEFORE paused_at to see the original paused_at value. + +-- name: QueuePauseAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END; + +-- name: QueuePauseByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +WHERE name = sqlc.arg('name'); + +-- name: QueueResumeAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL; + +-- name: QueueResumeByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +WHERE name = sqlc.arg('name'); + +-- name: QueueUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_queue +SET + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_update') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END, + updated_at = CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00') +WHERE name = sqlc.arg('name'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go new file mode 100644 index 000000000..68e9046ec --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go @@ -0,0 +1,321 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_queue.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" + "time" +) + +const queueCreateOrSetUpdatedAtExec = `-- name: QueueCreateOrSetUpdatedAtExec :exec +INSERT INTO /* TEMPLATE: schema */river_queue ( + created_at, + metadata, + name, + paused_at, + updated_at +) VALUES ( + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(? AS JSON), + ?, + ?, + COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +) ON DUPLICATE KEY UPDATE + updated_at = COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +` + +type QueueCreateOrSetUpdatedAtExecParams struct { + Now sql.NullTime + Metadata []byte + Name string + PausedAt sql.NullTime + UpdatedAt sql.NullTime +} + +func (q *Queries) QueueCreateOrSetUpdatedAtExec(ctx context.Context, db DBTX, arg *QueueCreateOrSetUpdatedAtExecParams) error { + _, err := db.ExecContext(ctx, queueCreateOrSetUpdatedAtExec, + arg.Now, + arg.Metadata, + arg.Name, + arg.PausedAt, + arg.UpdatedAt, + arg.Now, + arg.UpdatedAt, + arg.Now, + ) + return err +} + +const queueDeleteExpiredExec = `-- name: QueueDeleteExpiredExec :exec +DELETE FROM /* TEMPLATE: schema */river_queue +WHERE name IN (/*SLICE:names*/?) +` + +func (q *Queries) QueueDeleteExpiredExec(ctx context.Context, db DBTX, names []string) error { + query := queueDeleteExpiredExec + var queryParams []interface{} + if len(names) > 0 { + for _, v := range names { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:names*/?", strings.Repeat(",?", len(names))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:names*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const queueDeleteExpiredSelect = `-- name: QueueDeleteExpiredSelect :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE updated_at < ? +ORDER BY name ASC +LIMIT ? +FOR UPDATE +` + +type QueueDeleteExpiredSelectParams struct { + UpdatedAtHorizon time.Time + Limit int32 +} + +func (q *Queries) QueueDeleteExpiredSelect(ctx context.Context, db DBTX, arg *QueueDeleteExpiredSelectParams) ([]string, error) { + rows, err := db.QueryContext(ctx, queueDeleteExpiredSelect, arg.UpdatedAtHorizon, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + items = append(items, name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queueGet = `-- name: QueueGet :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = ? +` + +func (q *Queries) QueueGet(ctx context.Context, db DBTX, name string) (*RiverQueue, error) { + row := db.QueryRowContext(ctx, queueGet, name) + var i RiverQueue + err := row.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ) + return &i, err +} + +const queueGetForUpdate = `-- name: QueueGetForUpdate :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = ? +FOR UPDATE +` + +func (q *Queries) QueueGetForUpdate(ctx context.Context, db DBTX, name string) (*RiverQueue, error) { + row := db.QueryRowContext(ctx, queueGetForUpdate, name) + var i RiverQueue + err := row.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ) + return &i, err +} + +const queueList = `-- name: QueueList :many +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +ORDER BY name ASC +LIMIT ? +` + +func (q *Queries) QueueList(ctx context.Context, db DBTX, limit int32) ([]*RiverQueue, error) { + rows, err := db.QueryContext(ctx, queueList, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverQueue + for rows.Next() { + var i RiverQueue + if err := rows.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queueNameList = `-- name: QueueNameList :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE + name > ? + AND (? = '' OR LOWER(name) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(?), '%')) + AND name NOT IN (/*SLICE:exclude*/?) +ORDER BY name ASC +LIMIT ? +` + +type QueueNameListParams struct { + After string + Match string + Exclude []string + Limit int32 +} + +func (q *Queries) QueueNameList(ctx context.Context, db DBTX, arg *QueueNameListParams) ([]string, error) { + query := queueNameList + var queryParams []interface{} + queryParams = append(queryParams, arg.After) + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.Match) + if len(arg.Exclude) > 0 { + for _, v := range arg.Exclude { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:exclude*/?", strings.Repeat(",?", len(arg.Exclude))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:exclude*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + items = append(items, name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queuePauseAll = `-- name: QueuePauseAll :execresult + +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +` + +type QueuePauseAllParams struct { + Now sql.NullTime +} + +// MySQL evaluates SET clauses left-to-right using already-updated values, so +// updated_at must be set BEFORE paused_at to see the original paused_at value. +func (q *Queries) QueuePauseAll(ctx context.Context, db DBTX, arg *QueuePauseAllParams) (sql.Result, error) { + return db.ExecContext(ctx, queuePauseAll, arg.Now, arg.Now) +} + +const queuePauseByName = `-- name: QueuePauseByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +WHERE name = ? +` + +type QueuePauseByNameParams struct { + Now sql.NullTime + Name string +} + +func (q *Queries) QueuePauseByName(ctx context.Context, db DBTX, arg *QueuePauseByNameParams) (sql.Result, error) { + return db.ExecContext(ctx, queuePauseByName, arg.Now, arg.Now, arg.Name) +} + +const queueResumeAll = `-- name: QueueResumeAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +` + +func (q *Queries) QueueResumeAll(ctx context.Context, db DBTX, now sql.NullTime) (sql.Result, error) { + return db.ExecContext(ctx, queueResumeAll, now) +} + +const queueResumeByName = `-- name: QueueResumeByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +WHERE name = ? +` + +type QueueResumeByNameParams struct { + Now sql.NullTime + Name string +} + +func (q *Queries) QueueResumeByName(ctx context.Context, db DBTX, arg *QueueResumeByNameParams) (sql.Result, error) { + return db.ExecContext(ctx, queueResumeByName, arg.Now, arg.Name) +} + +const queueUpdateExec = `-- name: QueueUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_queue +SET + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END, + updated_at = CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00') +WHERE name = ? +` + +type QueueUpdateExecParams struct { + MetadataDoUpdate int64 + Metadata []byte + Name string +} + +func (q *Queries) QueueUpdateExec(ctx context.Context, db DBTX, arg *QueueUpdateExecParams) error { + _, err := db.ExecContext(ctx, queueUpdateExec, arg.MetadataDoUpdate, arg.Metadata, arg.Name) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/schema.sql b/riverdriver/rivermysql/internal/dbsqlc/schema.sql new file mode 100644 index 000000000..877f9460c --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/schema.sql @@ -0,0 +1,79 @@ +-- Dummy table definitions for INFORMATION_SCHEMA system tables so sqlc can +-- resolve column types. At runtime the template prefix replaces the empty +-- default with "INFORMATION_SCHEMA.", making queries target the real system +-- tables. +CREATE TABLE COLUMNS ( + COLUMN_NAME VARCHAR(128) NOT NULL, + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +CREATE TABLE SCHEMATA ( + SCHEMA_NAME VARCHAR(128) NOT NULL +); + +CREATE TABLE STATISTICS ( + INDEX_NAME VARCHAR(128) NOT NULL, + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +CREATE TABLE TABLES ( + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +-- name: ColumnExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */COLUMNS + WHERE COLUMN_NAME = sqlc.arg('column_name') + AND TABLE_NAME = sqlc.arg('table_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); + +-- name: IndexExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */STATISTICS + WHERE INDEX_NAME = sqlc.arg('index_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); + +-- name: IndexGetTableName :one +SELECT TABLE_NAME +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME = sqlc.arg('index_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +LIMIT 1; + +-- name: IndexReindexArtifacts :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) + AND REGEXP_LIKE(INDEX_NAME, sqlc.arg('artifact_pattern'), 'c') +ORDER BY INDEX_NAME; + +-- name: IndexesExist :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME IN (sqlc.slice('index_names')) + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()); + +-- name: Ping :one +SELECT 1; + +-- name: SchemaGetExpired :many +SELECT SCHEMA_NAME +FROM /* TEMPLATE: information_schema */SCHEMATA +WHERE SCHEMA_NAME LIKE CONCAT(sqlc.arg('prefix'), '%') + AND SCHEMA_NAME < sqlc.arg('before_name') +ORDER BY SCHEMA_NAME; + +-- name: TableExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */TABLES + WHERE TABLE_NAME = sqlc.arg('table_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); diff --git a/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go b/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go new file mode 100644 index 000000000..d1ea7467e --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go @@ -0,0 +1,226 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: schema.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" +) + +const columnExists = `-- name: ColumnExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */COLUMNS + WHERE COLUMN_NAME = ? + AND TABLE_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type ColumnExistsParams struct { + ColumnName string + TableName string + Schema sql.NullString +} + +func (q *Queries) ColumnExists(ctx context.Context, db DBTX, arg *ColumnExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, columnExists, arg.ColumnName, arg.TableName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const indexExists = `-- name: IndexExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */STATISTICS + WHERE INDEX_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type IndexExistsParams struct { + IndexName string + Schema sql.NullString +} + +func (q *Queries) IndexExists(ctx context.Context, db DBTX, arg *IndexExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, indexExists, arg.IndexName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const indexGetTableName = `-- name: IndexGetTableName :one +SELECT TABLE_NAME +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +LIMIT 1 +` + +type IndexGetTableNameParams struct { + IndexName string + Schema sql.NullString +} + +func (q *Queries) IndexGetTableName(ctx context.Context, db DBTX, arg *IndexGetTableNameParams) (string, error) { + row := db.QueryRowContext(ctx, indexGetTableName, arg.IndexName, arg.Schema) + var table_name string + err := row.Scan(&table_name) + return table_name, err +} + +const indexReindexArtifacts = `-- name: IndexReindexArtifacts :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) + AND REGEXP_LIKE(INDEX_NAME, ?, 'c') +ORDER BY INDEX_NAME +` + +type IndexReindexArtifactsParams struct { + Schema sql.NullString + ArtifactPattern string +} + +func (q *Queries) IndexReindexArtifacts(ctx context.Context, db DBTX, arg *IndexReindexArtifactsParams) ([]string, error) { + rows, err := db.QueryContext(ctx, indexReindexArtifacts, arg.Schema, arg.ArtifactPattern) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var index_name string + if err := rows.Scan(&index_name); err != nil { + return nil, err + } + items = append(items, index_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const indexesExist = `-- name: IndexesExist :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME IN (/*SLICE:index_names*/?) + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +` + +type IndexesExistParams struct { + IndexNames []string + Schema sql.NullString +} + +func (q *Queries) IndexesExist(ctx context.Context, db DBTX, arg *IndexesExistParams) ([]string, error) { + query := indexesExist + var queryParams []interface{} + if len(arg.IndexNames) > 0 { + for _, v := range arg.IndexNames { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:index_names*/?", strings.Repeat(",?", len(arg.IndexNames))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:index_names*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Schema) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var index_name string + if err := rows.Scan(&index_name); err != nil { + return nil, err + } + items = append(items, index_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const ping = `-- name: Ping :one +SELECT 1 +` + +func (q *Queries) Ping(ctx context.Context, db DBTX) (int64, error) { + row := db.QueryRowContext(ctx, ping) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const schemaGetExpired = `-- name: SchemaGetExpired :many +SELECT SCHEMA_NAME +FROM /* TEMPLATE: information_schema */SCHEMATA +WHERE SCHEMA_NAME LIKE CONCAT(?, '%') + AND SCHEMA_NAME < ? +ORDER BY SCHEMA_NAME +` + +type SchemaGetExpiredParams struct { + Prefix interface{} + BeforeName string +} + +func (q *Queries) SchemaGetExpired(ctx context.Context, db DBTX, arg *SchemaGetExpiredParams) ([]string, error) { + rows, err := db.QueryContext(ctx, schemaGetExpired, arg.Prefix, arg.BeforeName) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var schema_name string + if err := rows.Scan(&schema_name); err != nil { + return nil, err + } + items = append(items, schema_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const tableExists = `-- name: TableExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */TABLES + WHERE TABLE_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type TableExistsParams struct { + TableName string + Schema sql.NullString +} + +func (q *Queries) TableExists(ctx context.Context, db DBTX, arg *TableExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, tableExists, arg.TableName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml b/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml new file mode 100644 index 000000000..9ddeb2276 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml @@ -0,0 +1,41 @@ +version: "2" +sql: + - engine: "mysql" + queries: + - river_job.sql + - river_leader.sql + - river_migration.sql + - river_notification.sql + - river_queue.sql + - schema.sql + schema: + - river_job.sql + - river_leader.sql + - river_migration.sql + - river_notification.sql + - river_queue.sql + - schema.sql + gen: + go: + package: "dbsqlc" + out: "." + emit_exact_table_names: true + emit_methods_with_db_argument: true + emit_params_struct_pointers: true + emit_pointers_for_null_types: true + emit_result_struct_pointers: true + + rename: + ids: "IDs" + ttl: "TTL" + + overrides: + - db_type: "json" + go_type: + type: "[]byte" + - db_type: "json" + go_type: + type: "[]byte" + nullable: true + - db_type: "int" + go_type: "int64" diff --git a/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql b/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql b/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..9aee5ff2f --- /dev/null +++ b/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql b/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql b/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql new file mode 100644 index 000000000..c680cb3db --- /dev/null +++ b/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify is +-- omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB; diff --git a/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql b/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql b/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..7725c9f12 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. MySQL doesn't have enum types to +-- alter, so we update the CHECK constraint instead. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql b/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..108c807a8 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql b/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..fc5889c45 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; diff --git a/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql b/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql b/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..c4c740ea7 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..d5dc9a6b4 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/riverdriver/rivermysql/river_mysql_driver.go b/riverdriver/rivermysql/river_mysql_driver.go new file mode 100644 index 000000000..0a42d6c56 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver.go @@ -0,0 +1,2211 @@ +// Package rivermysql provides a River driver implementation for MySQL. +// +// This driver targets MySQL 8.0+ and requires `go-sql-driver/mysql` or a +// compatible driver registered with `database/sql`. DSNs used with River +// should include `parseTime=true`, `loc=UTC`, and `multiStatements=true`. +// `parseTime` and `loc` ensure `DATETIME` columns are scanned as UTC +// [time.Time] values, while `multiStatements` is needed to run River's +// migrations. +// +// MySQL does not support LISTEN/NOTIFY, so this driver uses a +// `river_notification` outbox table for notifications. It also does not support +// `RETURNING` clauses, so most write operations are carried out as two-step +// operations (write + read). +// +// This driver is currently in early development. It's exercised in the test +// suite, but has minimal real world use as of yet. +package rivermysql + +import ( + "bytes" + "context" + "database/sql" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" + "regexp" + "slices" + "strings" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" + "github.com/riverqueue/river/rivershared/uniquestates" + "github.com/riverqueue/river/rivershared/util/dbutil" + "github.com/riverqueue/river/rivershared/util/randutil" + "github.com/riverqueue/river/rivershared/util/savepointutil" + "github.com/riverqueue/river/rivershared/util/sliceutil" + "github.com/riverqueue/river/rivertype" +) + +//go:embed migration/*/*.sql +var migrationFS embed.FS + +// Driver is an implementation of riverdriver.Driver for MySQL. +type Driver struct { + dbPool *sql.DB + replacer sqlctemplate.Replacer +} + +// New returns a new MySQL driver for use with River. +// +// It takes an sql.DB to use with River. The DSN should include +// `parseTime=true&loc=UTC&multiStatements=true`; see the package documentation +// for details. The pool must not be closed while associated River objects are +// running. +func New(dbPool *sql.DB) *Driver { + return &Driver{ + dbPool: dbPool, + replacer: sqlctemplate.Replacer{UnnumberedPlaceholders: true}, + } +} + +const argPlaceholder = "?" + +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameMySQL } +func (d *Driver) SafeIdentifier(ident string) string { return mysqlIdentifier(ident) } + +func (d *Driver) GetExecutor() riverdriver.Executor { + return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d, nil} +} + +func (d *Driver) GetListener(params *riverdriver.GetListenenerParams) riverdriver.Listener { + return &Listener{ + dbPool: d.dbPool, + pollInterval: notificationPollIntervalDefault, + replacer: &d.replacer, + schema: params.Schema, + topics: make(map[string]int64), + } +} + +func (d *Driver) GetMigrationDefaultLines() []string { return []string{riverdriver.MigrationLineMain} } + +func (d *Driver) GetMigrationFS(line string) fs.FS { + if line == riverdriver.MigrationLineMain { + return migrationFS + } + panic("migration line does not exist: " + line) +} +func (d *Driver) GetMigrationLines() []string { return []string{riverdriver.MigrationLineMain} } +func (d *Driver) GetMigrationTruncateTables(line string, version int) []string { + if line == riverdriver.MigrationLineMain { + return riverdriver.MigrationLineMainTruncateTables(version) + } + panic("migration line does not exist: " + line) +} + +func (d *Driver) PoolIsSet() bool { return d.dbPool != nil } +func (d *Driver) PoolSet(dbPool any) error { + if d.dbPool != nil { + return errors.New("cannot PoolSet when internal pool is already non-nil") + } + d.dbPool = dbPool.(*sql.DB) //nolint:forcetypeassert + return nil +} + +func (d *Driver) SQLFragmentColumnContainsAll(column, namedArg string, values []string) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + return fmt.Sprintf("JSON_CONTAINS(%s, CAST(@%s AS JSON))", column, namedArg), arg, nil +} + +func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + return fmt.Sprintf("EXISTS (SELECT 1 FROM JSON_TABLE(CAST(@%s AS JSON), '$[*]' COLUMNS(val VARCHAR(255) PATH '$')) AS filter_value WHERE JSON_CONTAINS(%s, JSON_QUOTE(filter_value.val)))", namedArg, column), arg, nil +} + +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + if !json.Valid([]byte(fragment)) { + return "", nil, errors.New("invalid JSON metadata fragment") + } + args := make(map[string]any) + bindFunc := func(value any) string { + name := fmt.Sprintf("%s_%d", namedArg, len(args)) + args[name] = value + return "@" + name + } + predicate, err := jsonContains(column, json.RawMessage(fragment), bindFunc, 0) + return predicate, args, err +} + +func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + + // Match integer types to avoid implicit string-to-double conversion losing + // precision for large IDs. Strings use River's case-sensitive collation. + valueType := "VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" + switch values.(type) { + case []int, []int8, []int16, []int32, []int64: + valueType = "BIGINT" + } + + return fmt.Sprintf("%s IN (SELECT jt.val FROM JSON_TABLE(CAST(@%s AS JSON), '$[*]' COLUMNS(val %s PATH '$')) AS jt)", column, column, valueType), arg, nil +} + +func (d *Driver) SupportsListener() bool { return true } +func (d *Driver) SupportsListenNotify() bool { return true } +func (d *Driver) TimePrecision() time.Duration { return time.Microsecond } + +func (d *Driver) UnwrapExecutor(tx *sql.Tx) riverdriver.ExecutorTx { + // Allows UnwrapExecutor to be invoked even if driver is nil. + var replacer *sqlctemplate.Replacer + if d == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } else { + replacer = &d.replacer + } + + executorTx := &ExecutorTx{tx: tx} + executorTx.Executor = Executor{nil, templateReplaceWrapper{tx, replacer}, d, executorTx} + + return executorTx +} + +func (d *Driver) UnwrapTx(execTx riverdriver.ExecutorTx) *sql.Tx { + switch execTx := execTx.(type) { + case *ExecutorSubTx: + return execTx.tx + case *ExecutorTx: + return execTx.tx + } + panic("unhandled executor type") +} + +type Executor struct { + dbPool *sql.DB + dbtx templateReplaceWrapper + driver *Driver + execTx riverdriver.ExecutorTx +} + +func (e *Executor) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + if e.execTx != nil { + return e.execTx.Begin(ctx) + } + + tx, err := e.dbPool.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + + executorTx := &ExecutorTx{tx: tx} + executorTx.Executor = Executor{nil, templateReplaceWrapper{tx, &e.driver.replacer}, e.driver, executorTx} + return executorTx, nil +} + +func (e *Executor) ColumnExists(ctx context.Context, params *riverdriver.ColumnExistsParams) (bool, error) { + exists, err := dbsqlc.New().ColumnExists(informationSchemaParam(ctx), e.dbtx, &dbsqlc.ColumnExistsParams{ + ColumnName: params.Column, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + TableName: params.Table, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) Exec(ctx context.Context, sql string, args ...any) error { + _, err := e.dbtx.ExecContext(ctx, sql, args...) + return interpretError(err) +} + +func (e *Executor) IndexDropIfExists(ctx context.Context, params *riverdriver.IndexDropIfExistsParams) error { + indexName := strings.TrimSpace(params.Index) + + exists, err := e.IndexExists(ctx, &riverdriver.IndexExistsParams{ + Index: indexName, + Schema: params.Schema, + }) + if err != nil { + return err + } + if !exists { + return nil + } + + // MySQL's DROP INDEX requires the table name. Look it up from the index. + tableName, err := dbsqlc.New().IndexGetTableName(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexGetTableNameParams{ + IndexName: indexName, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return interpretError(err) + } + + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + _, err = e.dbtx.ExecContext(ctx, "DROP INDEX "+mysqlIdentifier(indexName)+" ON "+maybeSchema+mysqlIdentifier(tableName)) + return interpretError(err) +} + +func (e *Executor) IndexExists(ctx context.Context, params *riverdriver.IndexExistsParams) (bool, error) { + ctx = informationSchemaParam(ctx) + + exists, err := dbsqlc.New().IndexExists(ctx, e.dbtx, &dbsqlc.IndexExistsParams{ + IndexName: params.Index, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) IndexReindex(ctx context.Context, params *riverdriver.IndexReindexParams) error { + // MySQL has no per-index REINDEX operation. ANALYZE TABLE is the safe + // online approximation: it refreshes the selected index's table statistics + // without rebuilding and locking the entire table as OPTIMIZE TABLE would. + tableName, err := dbsqlc.New().IndexGetTableName(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexGetTableNameParams{ + IndexName: params.Index, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return interpretError(err) + } + + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + _, err = e.dbtx.ExecContext(ctx, "ANALYZE TABLE "+maybeSchema+mysqlIdentifier(tableName)) + return interpretError(err) +} + +func (e *Executor) IndexReindexArtifacts(ctx context.Context, params *riverdriver.IndexReindexArtifactsParams) ([]string, error) { + artifacts, err := dbsqlc.New().IndexReindexArtifacts(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexReindexArtifactsParams{ + ArtifactPattern: "^" + regexp.QuoteMeta(params.Index) + `_(ccnew|ccold)[0-9]*$`, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + return artifacts, interpretError(err) +} + +func (e *Executor) IndexesExist(ctx context.Context, params *riverdriver.IndexesExistParams) (map[string]bool, error) { + foundNames, err := dbsqlc.New().IndexesExist(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexesExistParams{ + IndexNames: params.IndexNames, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return nil, interpretError(err) + } + + exists := make(map[string]bool, len(params.IndexNames)) + for _, name := range foundNames { + exists[name] = true + } + return exists, nil +} + +func (e *Executor) InitDriver(ctx context.Context) error { return nil } + +func (e *Executor) JobCancel(ctx context.Context, params *riverdriver.JobCancelParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + res, err := dbsqlc.New().JobCancelExec(ctx, dbtx, &dbsqlc.JobCancelExecParams{ + ID: params.ID, + CancelAttemptedAt: params.CancelAttemptedAt.UTC().Format(time.RFC3339Nano), + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + if rowsAffected < 1 { + // No rows were updated, return the job as-is (already finalized) + return jobRowFromInternal(job) + } + + if params.ControlTopic != "" { + if err := dbsqlc.New().NotificationInsertJobCancel(ctx, dbtx, &dbsqlc.NotificationInsertJobCancelParams{ + JobID: job.ID, + Queue: job.Queue, + Topic: params.ControlTopic, + }); err != nil { + return nil, fmt.Errorf("error inserting job cancellation notification: %w", interpretError(err)) + } + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobCountByAllStates(ctx context.Context, params *riverdriver.JobCountByAllStatesParams) (map[rivertype.JobState]int, error) { + counts, err := dbsqlc.New().JobCountByAllStates(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + countsMap := make(map[rivertype.JobState]int) + for _, state := range rivertype.JobStates() { + countsMap[state] = 0 + } + for _, count := range counts { + countsMap[rivertype.JobState(count.State)] = int(count.Count) + } + return countsMap, nil +} + +func (e *Executor) JobCountByQueueAndState(ctx context.Context, params *riverdriver.JobCountByQueueAndStateParams) ([]*riverdriver.JobCountByQueueAndStateResult, error) { + rows, err := dbsqlc.New().JobCountByQueueAndState(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.QueueNames) + if err != nil { + return nil, interpretError(err) + } + + // MySQL's GROUP BY only returns queues that have jobs, so fill in zero + // counts for any requested queues not in the result. + countsByQueue := make(map[string]*riverdriver.JobCountByQueueAndStateResult, len(rows)) + for _, row := range rows { + countsByQueue[row.Queue] = &riverdriver.JobCountByQueueAndStateResult{ + CountAvailable: row.CountAvailable, + CountRunning: row.CountRunning, + Queue: row.Queue, + } + } + + var ( + queueNames = slices.Compact(slices.Sorted(slices.Values(params.QueueNames))) + results = make([]*riverdriver.JobCountByQueueAndStateResult, len(queueNames)) + ) + for i, name := range queueNames { + if result, ok := countsByQueue[name]; ok { + results[i] = result + } else { + results[i] = &riverdriver.JobCountByQueueAndStateResult{Queue: name} + } + } + + return results, nil +} + +func (e *Executor) JobCountByState(ctx context.Context, params *riverdriver.JobCountByStateParams) (int, error) { + numJobs, err := dbsqlc.New().JobCountByState(schemaTemplateParam(ctx, params.Schema), e.dbtx, string(params.State)) + if err != nil { + return 0, err + } + return int(numJobs), nil +} + +func (e *Executor) JobDelete(ctx context.Context, params *riverdriver.JobDeleteParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Lock the job while reading it so a concurrent producer can't change it + // to running between the read and delete statements. + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + res, err := dbsqlc.New().JobDeleteExec(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, interpretError(err) + } + + if rowsAffected < 1 { + if rivertype.JobState(job.State) == rivertype.JobStateRunning { + return nil, rivertype.ErrJobRunning + } + return nil, fmt.Errorf("bug; expected only to fetch a job with state %q, but was: %q", rivertype.JobStateRunning, job.State) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobDeleteBefore(ctx context.Context, params *riverdriver.JobDeleteBeforeParams) (int, error) { + queuesExcluded := params.QueuesExcluded + if len(queuesExcluded) < 1 { + queuesExcluded = []string{""} + } + + queuesIncluded := params.QueuesIncluded + if len(queuesIncluded) < 1 { + queuesIncluded = []string{""} + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (int, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + ids, err := dbsqlc.New().JobDeleteBeforeSelect(ctx, dbtx, &dbsqlc.JobDeleteBeforeSelectParams{ + CancelledDoDelete: boolToInt64(params.CancelledDoDelete), + CancelledFinalizedAtHorizon: sql.NullTime{Time: params.CancelledFinalizedAtHorizon.UTC(), Valid: true}, + CompletedDoDelete: boolToInt64(params.CompletedDoDelete), + CompletedFinalizedAtHorizon: sql.NullTime{Time: params.CompletedFinalizedAtHorizon.UTC(), Valid: true}, + DiscardedDoDelete: boolToInt64(params.DiscardedDoDelete), + DiscardedFinalizedAtHorizon: sql.NullTime{Time: params.DiscardedFinalizedAtHorizon.UTC(), Valid: true}, + Limit: int32(params.Max), //nolint:gosec + QueuesExcluded: queuesExcluded, + QueuesExcludedEmpty: boolToInt64(len(params.QueuesExcluded) < 1), + QueuesIncluded: queuesIncluded, + QueuesIncludedEmpty: boolToInt64(params.QueuesIncluded == nil), + }) + if err != nil { + return 0, interpretError(err) + } + if len(ids) > 0 { + if err := dbsqlc.New().JobDeleteManyExec(ctx, dbtx, ids); err != nil { + return 0, interpretError(err) + } + } + return len(ids), nil + }) +} + +func (e *Executor) JobDeleteMany(ctx context.Context, params *riverdriver.JobDeleteManyParams) ([]*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*rivertype.JobRow, error) { + baseCtx := ctx + selectCtx := sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "order_by_clause": {Value: params.OrderByClause}, + "where_clause": {Value: params.WhereClause}, + }, params.NamedArgs) + selectCtx = schemaTemplateParam(selectCtx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Select and lock the rows before deleting them so that a producer can't + // start one of the selected jobs between the two statements. + jobs, err := dbsqlc.New().JobDeleteManySelect(selectCtx, dbtx, params.Max) + if err != nil { + return nil, interpretError(err) + } + + if len(jobs) > 0 { + // Use a fresh context with only the schema replacement. The select + // context has templates that aren't present in the DELETE query. + deleteCtx := schemaTemplateParam(baseCtx, params.Schema) + ids := sliceutil.Map(jobs, func(j *dbsqlc.RiverJob) int64 { return j.ID }) + if err := dbsqlc.New().JobDeleteManyExec(deleteCtx, dbtx, ids); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.MapError(jobs, jobRowFromInternal) + }) +} + +func (e *Executor) JobGetAvailable(ctx context.Context, params *riverdriver.JobGetAvailableParams) (*riverdriver.JobGetAvailableResult, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.JobGetAvailableResult, error) { + baseCtx := ctx + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Preserve byte-exact kind matching (including trailing spaces). sqlc + // loses slice arguments when the IN operand is a CAST expression. + selectCtx := sqlctemplate.WithReplacements(baseCtx, map[string]sqlctemplate.Replacement{ + "kind_column": {Stable: true, Value: "CAST(kind AS BINARY)"}, + }, nil) + selectCtx = schemaTemplateParam(selectCtx, params.Schema) + + ids, err := dbsqlc.New().JobGetAvailableIDs(selectCtx, dbtx, &dbsqlc.JobGetAvailableIDsParams{ + Kind: params.Kind, + KindFilter: boolToInt64(params.Kind != nil), + Queue: params.Queue, + Now: nullTimeFromPtr(params.Now), + Limit: int32(params.MaxToLock), //nolint:gosec + }) + if err != nil { + return nil, interpretError(err) + } + + if len(ids) == 0 { + return &riverdriver.JobGetAvailableResult{}, nil + } + + if err := dbsqlc.New().JobGetAvailableUpdate(ctx, dbtx, &dbsqlc.JobGetAvailableUpdateParams{ + Now: nullTimeFromPtr(params.Now), + MaxAttemptedBy: int64(params.MaxAttemptedBy), + AttemptedBy: params.ClientID, + ID: ids, + }); err != nil { + return nil, interpretError(err) + } + + jobs, err := dbsqlc.New().JobGetByIDManyOrdered(ctx, dbtx, ids) + if err != nil { + return nil, interpretError(err) + } + + return jobGetAvailableResultFromInternal(jobs), nil + }) +} + +func (e *Executor) JobGetByID(ctx context.Context, params *riverdriver.JobGetByIDParams) (*rivertype.JobRow, error) { + job, err := dbsqlc.New().JobGetByID(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) +} + +func (e *Executor) JobGetByIDMany(ctx context.Context, params *riverdriver.JobGetByIDManyParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetByIDMany(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobGetByKindMany(ctx context.Context, params *riverdriver.JobGetByKindManyParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetByKindMany(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Kind) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobGetCancelRequested(ctx context.Context, params *riverdriver.JobGetCancelRequestedParams) ([]int64, error) { + ids, err := dbsqlc.New().JobGetCancelRequested(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + return ids, interpretError(err) +} + +func (e *Executor) JobGetStuck(ctx context.Context, params *riverdriver.JobGetStuckParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetStuck(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.JobGetStuckParams{ + AfterID: params.AfterID, + Limit: int32(params.Max), //nolint:gosec + StuckHorizon: sql.NullTime{Time: params.StuckHorizon.UTC(), Valid: true}, + }) + if err != nil { + return nil, interpretError(err) + } + return jobRowsFromInternalPartial(jobs), nil +} + +func (e *Executor) JobInsertFastMany(ctx context.Context, params *riverdriver.JobInsertFastManyParams) ([]*riverdriver.JobInsertFastResult, error) { + // Match Postgres's rejection of a batch that affects one unique row twice. + uniqueKeysInBatch := make(map[string]bool, len(params.Jobs)) + for _, job := range params.Jobs { + if len(job.UniqueKey) > 0 && job.UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{job.State}) != 0 { + key := string(job.UniqueKey) + if uniqueKeysInBatch[key] { + return nil, errors.New("unique key appears more than once in batch") + } + uniqueKeysInBatch[key] = true + } + } + + var ( + insertRes = make([]*riverdriver.JobInsertFastResult, len(params.Jobs)) + uniqueNonceByIndex = make([]string, len(params.Jobs)) + ) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + baseCtx := ctx + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + var ( + idsWithNonce []int64 + jobs []*dbsqlc.RiverJob + ) + if jobInsertBatchSupported(params) { + var err error + jobs, err = jobInsertBatchReturning(baseCtx, dbtx, params, uniqueNonceByIndex) + if err != nil { + return err + } + idsWithNonce = sliceutil.Map(jobs, func(job *dbsqlc.RiverJob) int64 { return job.ID }) + } else { + plainInsertCtx := schemaTemplateParam(baseCtx, params.Schema) + plainInsertCtx = sqlctemplate.WithReplacements(plainInsertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: ""}, + }, nil) + uniqueInsertCtx := schemaTemplateParam(baseCtx, params.Schema) + uniqueInsertCtx = sqlctemplate.WithReplacements(uniqueInsertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: "ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id)"}, + }, nil) + + ids := make([]int64, len(params.Jobs)) + for i, jobParams := range params.Jobs { + // Only use MySQL's broad ON DUPLICATE KEY behavior when a River unique + // key is present. Otherwise, a primary key conflict or a user-added + // unique constraint must be returned as an error. + insertCtx := plainInsertCtx + if len(jobParams.UniqueKey) > 0 { + insertCtx = uniqueInsertCtx + + // Postgres can inspect xmax to determine whether an upsert inserted a + // row. MySQL has no equivalent, so add a per-row nonce to metadata and + // compare it with the selected result. Each input needs its own nonce + // so duplicates within this batch are classified correctly. + uniqueNonceByIndex[i] = randutil.Hex(8) + } + + insertParams, err := jobInsertFastParams(jobParams, uniqueNonceByIndex[i]) + if err != nil { + return err + } + + res, err := dbsqlc.New().JobInsertFast(insertCtx, dbtx, insertParams) + if err != nil { + return interpretError(err) + } + ids[i], err = res.LastInsertId() + if err != nil { + return err + } + if uniqueNonceByIndex[i] != "" { + idsWithNonce = append(idsWithNonce, ids[i]) + } + } + + selected, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, ids) + if err != nil { + return interpretError(err) + } + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(selected)) + for _, job := range selected { + jobsByID[job.ID] = job + } + jobs = sliceutil.Map(ids, func(id int64) *dbsqlc.RiverJob { return jobsByID[id] }) + } + + for i, internal := range jobs { + job, err := jobRowFromInternal(internal) + if err != nil { + return err + } + + insertRes[i] = &riverdriver.JobInsertFastResult{Job: job} + if uniqueNonceByIndex[i] != "" { + uniqueSkippedAsDuplicate := gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).Str != uniqueNonceByIndex[i] + if uniqueSkippedAsDuplicate && (!bytes.Equal(job.UniqueKey, params.Jobs[i].UniqueKey) || + !slices.Contains(job.UniqueStates, job.State) || + params.Jobs[i].UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{params.Jobs[i].State}) == 0) { + // Unlike Postgres and SQLite, MySQL's ON DUPLICATE KEY + // cannot target River's unique-job index. Reject a conflict + // on any other unique constraint instead of silently treating + // the unrelated row as the requested unique job. + return fmt.Errorf("job insert at index %d conflicted on a non-River unique constraint", i) + } + insertRes[i].UniqueSkippedAsDuplicate = uniqueSkippedAsDuplicate + job.Metadata, err = sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) + if err != nil { + return fmt.Errorf("error removing unique nonce from returned job metadata: %w", err) + } + } + } + + if len(idsWithNonce) > 0 { + if err := dbsqlc.New().JobClearUniqueNonce(ctx, dbtx, idsWithNonce); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return nil, err + } + + return insertRes, nil +} + +func (e *Executor) JobInsertFastManyNoReturning(ctx context.Context, params *riverdriver.JobInsertFastManyParams) (int, error) { + // RowsAffected can't distinguish a no-op duplicate from an insert when the + // DSN enables clientFoundRows. Fall back to nonce-based duplicate detection + // for any batch where a conflict is possible. + if slices.ContainsFunc(params.Jobs, func(job *riverdriver.JobInsertFastParams) bool { + return len(job.UniqueKey) > 0 + }) { + results, err := e.JobInsertFastMany(ctx, params) + if err != nil { + return 0, err + } + + numInserted := 0 + for _, result := range results { + if !result.UniqueSkippedAsDuplicate { + numInserted++ + } + } + return numInserted, nil + } + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + baseCtx := ctx + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + insertCtx := schemaTemplateParam(baseCtx, params.Schema) + if len(params.Jobs) > 1 { + _, err := jobInsertBatch(insertCtx, dbtx, params, nil) + return err + } + insertCtx = sqlctemplate.WithReplacements(insertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: ""}, + }, nil) + + for _, jobParams := range params.Jobs { + insertParams, err := jobInsertFastParams(jobParams, "") + if err != nil { + return err + } + + if _, err := dbsqlc.New().JobInsertFast(insertCtx, dbtx, insertParams); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return 0, err + } + + return len(params.Jobs), nil +} + +// jobInsertFastParams builds the common insert parameters for a single job. +// If uniqueNonce is non-empty, it's set in the job's metadata for duplicate +// detection. +func jobInsertFastParams(params *riverdriver.JobInsertFastParams, uniqueNonce string) (*dbsqlc.JobInsertFastParams, error) { + metadata := sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")) + if uniqueNonce != "" { + var err error + metadata, err = sjson.SetBytes(metadata, riverdriver.UniqueInsertMetadataKey, uniqueNonce) + if err != nil { + return nil, fmt.Errorf("error adding unique nonce to job metadata: %w", err) + } + } + + tags, err := json.Marshal(params.Tags) + if err != nil { + return nil, err + } + + var uniqueStates sql.NullInt16 + if params.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(params.UniqueStates), Valid: true} + } + + var id sql.NullInt64 + if params.ID != nil { + id = sql.NullInt64{Int64: *params.ID, Valid: true} + } + + return &dbsqlc.JobInsertFastParams{ + ID: id, + Args: params.EncodedArgs, + CreatedAt: utcTimePtr(params.CreatedAt), + Kind: params.Kind, + MaxAttempts: int64(params.MaxAttempts), + Metadata: metadata, + Priority: int16(params.Priority), //nolint:gosec + Queue: params.Queue, + ScheduledAt: utcTimePtr(params.ScheduledAt), + State: string(params.State), + Tags: tags, + UniqueKey: nullStringFromBytes(params.UniqueKey), + UniqueStates: uniqueStates, + }, nil +} + +func (e *Executor) JobInsertFull(ctx context.Context, params *riverdriver.JobInsertFullParams) (*rivertype.JobRow, error) { + var attemptedBy []byte + if params.AttemptedBy != nil { + var err error + attemptedBy, err = json.Marshal(params.AttemptedBy) + if err != nil { + return nil, err + } + } + + var errorsData []byte + if len(params.Errors) > 0 { + var err error + errorsData, err = json.Marshal(sliceutil.Map(params.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return nil, err + } + } + + tags, err := json.Marshal(params.Tags) + if err != nil { + return nil, err + } + + var uniqueStates sql.NullInt16 + if params.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(params.UniqueStates), Valid: true} + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + lastInsertID, err := dbsqlc.New().JobInsertFullExec(ctx, dbtx, &dbsqlc.JobInsertFullExecParams{ + Attempt: int64(params.Attempt), + AttemptedAt: nullTimeFromPtr(params.AttemptedAt), + AttemptedBy: attemptedBy, + Args: params.EncodedArgs, + CreatedAt: utcTimePtr(params.CreatedAt), + Errors: errorsData, + FinalizedAt: nullTimeFromPtr(params.FinalizedAt), + Kind: params.Kind, + MaxAttempts: int64(params.MaxAttempts), + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + Priority: int16(params.Priority), //nolint:gosec + Queue: params.Queue, + ScheduledAt: utcTimePtr(params.ScheduledAt), + State: string(params.State), + Tags: tags, + UniqueKey: nullStringFromBytes(params.UniqueKey), + UniqueStates: uniqueStates, + }) + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, lastInsertID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobInsertFullMany(ctx context.Context, params *riverdriver.JobInsertFullManyParams) ([]*rivertype.JobRow, error) { + insertRes := make([]*rivertype.JobRow, len(params.Jobs)) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + ids := make([]int64, len(params.Jobs)) + + for i, jobParams := range params.Jobs { + var attemptedBy []byte + if jobParams.AttemptedBy != nil { + var err error + attemptedBy, err = json.Marshal(jobParams.AttemptedBy) + if err != nil { + return err + } + } + + var errorsData []byte + if len(jobParams.Errors) > 0 { + var err error + errorsData, err = json.Marshal(sliceutil.Map(jobParams.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return err + } + } + + tags, err := json.Marshal(jobParams.Tags) + if err != nil { + return err + } + + var uniqueStates sql.NullInt16 + if jobParams.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(jobParams.UniqueStates), Valid: true} + } + + ids[i], err = dbsqlc.New().JobInsertFullExec(ctx, dbtx, &dbsqlc.JobInsertFullExecParams{ + Attempt: int64(jobParams.Attempt), + AttemptedAt: nullTimeFromPtr(jobParams.AttemptedAt), + AttemptedBy: attemptedBy, + Args: jobParams.EncodedArgs, + CreatedAt: utcTimePtr(jobParams.CreatedAt), + Errors: errorsData, + FinalizedAt: nullTimeFromPtr(jobParams.FinalizedAt), + Kind: jobParams.Kind, + MaxAttempts: int64(jobParams.MaxAttempts), + Metadata: sliceutil.FirstNonEmpty(jobParams.Metadata, []byte("{}")), + Priority: int16(jobParams.Priority), //nolint:gosec + Queue: jobParams.Queue, + ScheduledAt: utcTimePtr(jobParams.ScheduledAt), + State: string(jobParams.State), + Tags: tags, + UniqueKey: nullStringFromBytes(jobParams.UniqueKey), + UniqueStates: uniqueStates, + }) + if err != nil { + return interpretError(err) + } + } + + jobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, ids) + if err != nil { + return interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(jobs)) + for _, j := range jobs { + jobsByID[j.ID] = j + } + + for i, id := range ids { + insertRes[i], err = jobRowFromInternal(jobsByID[id]) + if err != nil { + return err + } + } + + return nil + }); err != nil { + return nil, err + } + + return insertRes, nil +} + +func (e *Executor) JobKindList(ctx context.Context, params *riverdriver.JobKindListParams) ([]string, error) { + exclude := params.Exclude + if len(exclude) == 0 { + exclude = []string{""} + } + + kinds, err := dbsqlc.New().JobKindList(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.JobKindListParams{ + After: params.After, + Exclude: exclude, + Match: params.Match, + Limit: int32(min(params.Max, math.MaxInt32)), //nolint:gosec + }) + if err != nil { + return nil, interpretError(err) + } + return kinds, nil +} + +func (e *Executor) JobList(ctx context.Context, params *riverdriver.JobListParams) ([]*rivertype.JobRow, error) { + ctx = sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "order_by_clause": {Value: params.OrderByClause}, + "where_clause": {Value: params.WhereClause}, + }, params.NamedArgs) + + jobs, err := dbsqlc.New().JobList(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Max) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobRescueMany(ctx context.Context, params *riverdriver.JobRescueManyParams) (*struct{}, error) { + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if len(params.ID) > 1 && batchIDsUnique(params.ID) { + if err := jobRescueBatch(ctx, dbtx, params); !isNoDatabaseSelected(err) { + return err + } + } + + for i := range params.ID { + if err := dbsqlc.New().JobRescue(ctx, dbtx, &dbsqlc.JobRescueParams{ + ID: params.ID[i], + Error: params.Error[i], + FinalizedAt: nullTimeFromPtr(params.FinalizedAt[i]), + ScheduledAt: params.ScheduledAt[i].UTC(), + State: params.State[i], + StuckHorizon: sql.NullTime{Time: params.StuckHorizon.UTC(), Valid: true}, + }); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return nil, err + } + + return &struct{}{}, nil +} + +func (e *Executor) JobRetry(ctx context.Context, params *riverdriver.JobRetryParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + _, err := dbsqlc.New().JobRetryExec(ctx, dbtx, &dbsqlc.JobRetryExecParams{ + ID: params.ID, + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobSchedule(ctx context.Context, params *riverdriver.JobScheduleParams) ([]*riverdriver.JobScheduleResult, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.JobScheduleResult, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + scheduleResults, err := dbsqlc.New().JobSchedule(ctx, dbtx, &dbsqlc.JobScheduleParams{ + Limit: int32(params.Max), //nolint:gosec + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + var ( + allIDs []int64 + availIDs []int64 + discardIDs []int64 + discardSet = make(map[int64]bool) + ) + + for _, result := range scheduleResults { + allIDs = append(allIDs, result.ID) + if result.ConflictDiscarded != 0 { + discardIDs = append(discardIDs, result.ID) + discardSet[result.ID] = true + } else { + availIDs = append(availIDs, result.ID) + } + } + + if len(availIDs) > 0 { + if err := dbsqlc.New().JobScheduleSetAvailableExec(ctx, dbtx, availIDs); err != nil { + return nil, interpretError(err) + } + } + + if len(discardIDs) > 0 { + if err := dbsqlc.New().JobScheduleSetDiscardedExec(ctx, dbtx, &dbsqlc.JobScheduleSetDiscardedExecParams{ + ID: discardIDs, + Now: nullTimeFromPtr(params.Now), + }); err != nil { + return nil, interpretError(err) + } + } + + if len(allIDs) == 0 { + return nil, nil + } + + updatedJobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, allIDs) + if err != nil { + return nil, interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(updatedJobs)) + for _, j := range updatedJobs { + jobsByID[j.ID] = j + } + + // Return results in the same order as scheduleResults. + results := make([]*riverdriver.JobScheduleResult, len(scheduleResults)) + for i, sr := range scheduleResults { + job, _ := jobRowFromInternalPartial(jobsByID[sr.ID]) + results[i] = &riverdriver.JobScheduleResult{ConflictDiscarded: discardSet[sr.ID], Job: *job} + } + + return results, nil + }) +} + +func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdriver.JobSetStateIfRunningManyParams) ([]*rivertype.JobRow, error) { + setRes := make([]*rivertype.JobRow, 0, len(params.ID)) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + metadata, err := jobMetadataMergeMany(ctx, dbtx, params.ID, params.MetadataDoMerge, params.MetadataUpdates) + if err != nil { + return err + } + paramsCopy := *params + paramsCopy.MetadataUpdates = metadata + params := ¶msCopy + + batchDone := false + if len(params.ID) > 1 && batchIDsUnique(params.ID) { + err := jobSetStateBatch(ctx, dbtx, params) + if err != nil && !isNoDatabaseSelected(err) { + return fmt.Errorf("error setting job state: %w", err) + } + batchDone = err == nil + } + if !batchDone { + // Execute all state changes, including metadata-only updates to non-running jobs. + for i := range params.ID { + setStateParams := &dbsqlc.JobSetStateIfRunningExecParams{ + ID: params.ID[i], + Error: []byte("{}"), + MetadataUpdates: []byte("{}"), + Now: nullTimeFromPtr(params.Now), + State: string(params.State[i]), + } + + if params.Attempt[i] != nil { + setStateParams.AttemptDoUpdate = 1 + setStateParams.Attempt = int64(*params.Attempt[i]) + } + if params.ErrData[i] != nil { + setStateParams.ErrorsDoUpdate = 1 + setStateParams.Error = params.ErrData[i] + } + if params.FinalizedAt[i] != nil { + setStateParams.FinalizedAtDoUpdate = 1 + setStateParams.FinalizedAt = nullTimeFromPtr(params.FinalizedAt[i]) + } + if params.MetadataDoMerge[i] { + setStateParams.MetadataDoMerge = 1 + setStateParams.MetadataUpdates = params.MetadataUpdates[i] + } + if params.ScheduledAt[i] != nil { + setStateParams.ScheduledAtDoUpdate = 1 + setStateParams.ScheduledAt = params.ScheduledAt[i].UTC() + } + + if err := dbsqlc.New().JobSetStateIfRunningExec(ctx, dbtx, setStateParams); err != nil { + return fmt.Errorf("error setting job state: %w", err) + } + } + } + + // Use a current read to include jobs that another transaction already finalized. + jobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, params.ID) + if err != nil { + return interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(jobs)) + for _, j := range jobs { + jobsByID[j.ID] = j + } + + // Build results in original order. + for _, id := range params.ID { + job := jobsByID[id] + if job == nil { + continue + } + + jobRow, _ := jobRowFromInternalPartial(job) + setRes = append(setRes, jobRow) + } + + return nil + }); err != nil { + return nil, err + } + + return setRes, nil +} + +func (e *Executor) JobUpdate(ctx context.Context, params *riverdriver.JobUpdateParams) (*rivertype.JobRow, error) { + metadata := params.Metadata + if metadata == nil { + metadata = []byte("{}") + } + + var metadataDoMerge int64 + if params.MetadataDoMerge { + metadataDoMerge = 1 + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + merged, err := jobMetadataMergeMany(ctx, dbtx, []int64{params.ID}, []bool{params.MetadataDoMerge}, [][]byte{metadata}) + if err != nil { + return nil, err + } + if err := dbsqlc.New().JobUpdateExec(ctx, dbtx, &dbsqlc.JobUpdateExecParams{ + ID: params.ID, + MetadataDoMerge: metadataDoMerge, + Metadata: merged[0], + }); err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobUpdateFull(ctx context.Context, params *riverdriver.JobUpdateFullParams) (*rivertype.JobRow, error) { + attemptedAt := params.AttemptedAt + if attemptedAt != nil { + attemptedAt = new(attemptedAt.UTC()) + } + + attemptedBy, err := json.Marshal(params.AttemptedBy) + if err != nil { + return nil, err + } + + errorsData, err := json.Marshal(sliceutil.Map(params.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return nil, err + } + + finalizedAt := params.FinalizedAt + if finalizedAt != nil { + finalizedAt = new(finalizedAt.UTC()) + } + + metadata := params.Metadata + if metadata == nil { + metadata = []byte("{}") + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().JobUpdateFullExec(ctx, dbtx, &dbsqlc.JobUpdateFullExecParams{ + ID: params.ID, + Attempt: int64(params.Attempt), + AttemptDoUpdate: boolToInt64(params.AttemptDoUpdate), + AttemptedAt: nullTimeFromPtr(attemptedAt), + AttemptedAtDoUpdate: boolToInt64(params.AttemptedAtDoUpdate), + AttemptedBy: attemptedBy, + AttemptedByDoUpdate: boolToInt64(params.AttemptedByDoUpdate), + ErrorsDoUpdate: boolToInt64(params.ErrorsDoUpdate), + Errors: errorsData, + FinalizedAtDoUpdate: boolToInt64(params.FinalizedAtDoUpdate), + FinalizedAt: nullTimeFromPtr(finalizedAt), + MaxAttemptsDoUpdate: boolToInt64(params.MaxAttemptsDoUpdate), + MaxAttempts: int64(min(params.MaxAttempts, math.MaxInt64)), + MetadataDoUpdate: boolToInt64(params.MetadataDoUpdate), + Metadata: metadata, + StateDoUpdate: boolToInt64(params.StateDoUpdate), + State: string(params.State), + }); err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) LeaderAttemptElect(ctx context.Context, params *riverdriver.LeaderElectParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + err := dbsqlc.New().LeaderAttemptElectExec(ctx, dbtx, &dbsqlc.LeaderAttemptElectExecParams{ + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + TTL: params.TTL.Microseconds(), + }) + if err != nil { + if isDuplicateEntry(err) { + return nil, rivertype.ErrNotFound + } + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetElectedLeader(ctx, dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderAttemptReelect(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + err := dbsqlc.New().LeaderAttemptReelectExec(ctx, dbtx, &dbsqlc.LeaderAttemptReelectExecParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + TTL: params.TTL.Microseconds(), + }) + if err != nil { + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetReelected(ctx, dbtx, &dbsqlc.LeaderGetReelectedParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderDeleteExpired(ctx context.Context, params *riverdriver.LeaderDeleteExpiredParams) (int, error) { + numDeleted, err := dbsqlc.New().LeaderDeleteExpired(schemaTemplateParam(ctx, params.Schema), e.dbtx, nullTimeFromPtr(params.Now)) + if err != nil { + return 0, interpretError(err) + } + return int(numDeleted), nil +} + +func (e *Executor) LeaderGetElectedLeader(ctx context.Context, params *riverdriver.LeaderGetElectedLeaderParams) (*riverdriver.Leader, error) { + leader, err := dbsqlc.New().LeaderGetElectedLeader(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil +} + +func (e *Executor) LeaderInsert(ctx context.Context, params *riverdriver.LeaderInsertParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().LeaderInsertExec(ctx, dbtx, &dbsqlc.LeaderInsertExecParams{ + ElectedAt: utcTimePtr(params.ElectedAt), + ExpiresAt: utcTimePtr(params.ExpiresAt), + Now: nullTimeFromPtr(params.Now), + LeaderID: params.LeaderID, + TTL: params.TTL.Microseconds(), + }); err != nil { + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetElectedLeader(ctx, dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderResign(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) { + numResigned, err := dbsqlc.New().LeaderResign(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.LeaderResignParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + }) + if err != nil { + return false, interpretError(err) + } + return numResigned > 0, nil +} + +func (e *Executor) MigrationDeleteAssumingMainMany(ctx context.Context, params *riverdriver.MigrationDeleteAssumingMainManyParams) ([]*riverdriver.Migration, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.Migration, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + migrations, err := dbsqlc.New().RiverMigrationDeleteAssumingMainMany(ctx, dbtx, versions) + if err != nil { + return nil, interpretError(err) + } + + if len(versions) > 0 { + if err := dbsqlc.New().RiverMigrationDeleteAssumingMainManyExec(ctx, dbtx, versions); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.Map(migrations, func(internal *dbsqlc.RiverMigrationDeleteAssumingMainManyRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }), nil + }) +} + +func (e *Executor) MigrationDeleteByLineAndVersionMany(ctx context.Context, params *riverdriver.MigrationDeleteByLineAndVersionManyParams) ([]*riverdriver.Migration, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.Migration, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + migrations, err := dbsqlc.New().RiverMigrationDeleteByLineAndVersionMany(ctx, dbtx, &dbsqlc.RiverMigrationDeleteByLineAndVersionManyParams{ + Line: params.Line, + Version: versions, + }) + if err != nil { + return nil, interpretError(err) + } + + if len(versions) > 0 { + if err := dbsqlc.New().RiverMigrationDeleteByLineAndVersionManyExec(ctx, dbtx, &dbsqlc.RiverMigrationDeleteByLineAndVersionManyExecParams{ + Line: params.Line, + Version: versions, + }); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.Map(migrations, migrationFromInternal), nil + }) +} + +func (e *Executor) MigrationGetAllAssumingMain(ctx context.Context, params *riverdriver.MigrationGetAllAssumingMainParams) ([]*riverdriver.Migration, error) { + migrations, err := dbsqlc.New().RiverMigrationGetAllAssumingMain(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(migrations, func(internal *dbsqlc.RiverMigrationGetAllAssumingMainRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }), nil +} + +func (e *Executor) MigrationGetByLine(ctx context.Context, params *riverdriver.MigrationGetByLineParams) ([]*riverdriver.Migration, error) { + migrations, err := dbsqlc.New().RiverMigrationGetByLine(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Line) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(migrations, migrationFromInternal), nil +} + +func (e *Executor) MigrationInsertMany(ctx context.Context, params *riverdriver.MigrationInsertManyParams) ([]*riverdriver.Migration, error) { + var migrations []*riverdriver.Migration + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + for _, version := range params.Versions { + if err := dbsqlc.New().RiverMigrationInsertExec(ctx, dbtx, &dbsqlc.RiverMigrationInsertExecParams{ + Line: params.Line, + Version: int64(version), + }); err != nil { + return interpretError(err) + } + } + + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + internals, err := dbsqlc.New().RiverMigrationGetByLineAndVersionMany(ctx, dbtx, &dbsqlc.RiverMigrationGetByLineAndVersionManyParams{ + Line: params.Line, + Version: versions, + }) + if err != nil { + return interpretError(err) + } + + migrations = sliceutil.Map(internals, migrationFromInternal) + return nil + }); err != nil { + return nil, err + } + + return migrations, nil +} + +func (e *Executor) MigrationInsertManyAssumingMain(ctx context.Context, params *riverdriver.MigrationInsertManyAssumingMainParams) ([]*riverdriver.Migration, error) { + var migrations []*riverdriver.Migration + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + for _, version := range params.Versions { + if err := dbsqlc.New().RiverMigrationInsertAssumingMainExec(ctx, dbtx, int64(version)); err != nil { + return interpretError(err) + } + } + + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + internals, err := dbsqlc.New().RiverMigrationGetByVersionMany(ctx, dbtx, versions) + if err != nil { + return interpretError(err) + } + + migrations = sliceutil.Map(internals, func(internal *dbsqlc.RiverMigrationGetByVersionManyRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }) + return nil + }); err != nil { + return nil, err + } + + return migrations, nil +} + +func (e *Executor) NotificationDeleteBefore(ctx context.Context, params *riverdriver.NotificationDeleteBeforeParams) (int, error) { + numDeleted, err := dbsqlc.New().NotificationDeleteBefore(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.NotificationDeleteBeforeParams{CreatedAtHorizon: params.CreatedAtHorizon.UTC(), Limit: int32(params.Max)}) //nolint:gosec + return int(numDeleted), interpretError(err) +} + +func (e *Executor) NotifyMany(ctx context.Context, params *riverdriver.NotifyManyParams) error { + if len(params.Payload) < 1 { + return nil + } + + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if len(params.Payload) > 1 { + return execJSONBatch(ctx, dbtx, notificationInsertBatchSQL, params.Payload, params.Topic) + } + + for _, payload := range params.Payload { + if err := dbsqlc.New().NotificationInsert(ctx, dbtx, &dbsqlc.NotificationInsertParams{ + Payload: payload, + Topic: params.Topic, + }); err != nil { + return interpretError(err) + } + } + + return nil + }) +} + +func (e *Executor) Ping(ctx context.Context) error { + _, err := dbsqlc.New().Ping(ctx, e.dbtx) + return interpretError(err) +} + +func (e *Executor) PGAdvisoryXactLock(ctx context.Context, key int64) (*struct{}, error) { + // MySQL has GET_LOCK which is similar to Postgres advisory locks, but + // it's session-scoped rather than transaction-scoped. For now, return + // not implemented. + return nil, riverdriver.ErrNotImplemented +} + +func (e *Executor) QueueCreateOrSetUpdatedAt(ctx context.Context, params *riverdriver.QueueCreateOrSetUpdatedAtParams) (*rivertype.Queue, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.Queue, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().QueueCreateOrSetUpdatedAtExec(ctx, dbtx, &dbsqlc.QueueCreateOrSetUpdatedAtExecParams{ + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + Name: params.Name, + Now: nullTimeFromPtr(params.Now), + PausedAt: nullTimeFromPtr(params.PausedAt), + UpdatedAt: nullTimeFromPtr(params.UpdatedAt), + }); err != nil { + return nil, interpretError(err) + } + + queue, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil + }) +} + +func (e *Executor) QueueDeleteExpired(ctx context.Context, params *riverdriver.QueueDeleteExpiredParams) ([]string, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]string, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + queueNames, err := dbsqlc.New().QueueDeleteExpiredSelect(ctx, dbtx, &dbsqlc.QueueDeleteExpiredSelectParams{ + Limit: int32(params.Max), //nolint:gosec + UpdatedAtHorizon: params.UpdatedAtHorizon.UTC(), + }) + if err != nil { + return nil, interpretError(err) + } + + if len(queueNames) > 0 { + if err := dbsqlc.New().QueueDeleteExpiredExec(ctx, dbtx, queueNames); err != nil { + return nil, interpretError(err) + } + } + + return queueNames, nil + }) +} + +func (e *Executor) QueueGet(ctx context.Context, params *riverdriver.QueueGetParams) (*rivertype.Queue, error) { + queue, err := dbsqlc.New().QueueGet(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil +} + +func (e *Executor) QueueList(ctx context.Context, params *riverdriver.QueueListParams) ([]*rivertype.Queue, error) { + queues, err := dbsqlc.New().QueueList(schemaTemplateParam(ctx, params.Schema), e.dbtx, int32(params.Max)) //nolint:gosec + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(queues, queueFromInternal), nil +} + +func (e *Executor) QueueNameList(ctx context.Context, params *riverdriver.QueueNameListParams) ([]string, error) { + exclude := params.Exclude + if len(exclude) == 0 { + exclude = []string{""} + } + queueNames, err := dbsqlc.New().QueueNameList(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.QueueNameListParams{ + After: params.After, + Exclude: exclude, + Match: params.Match, + Limit: int32(min(params.Max, math.MaxInt32)), //nolint:gosec + }) + if err != nil { + return nil, interpretError(err) + } + return queueNames, nil +} + +func (e *Executor) QueuePause(ctx context.Context, params *riverdriver.QueuePauseParams) error { + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + now := nullTimeFromPtr(params.Now) + + var ( + res sql.Result + err error + ) + + if params.Name == riverdriver.AllQueuesString { + res, err = dbsqlc.New().QueuePauseAll(ctx, dbtx, &dbsqlc.QueuePauseAllParams{Now: now}) + } else { + res, err = dbsqlc.New().QueuePauseByName(ctx, dbtx, &dbsqlc.QueuePauseByNameParams{ + Name: params.Name, + Now: now, + }) + } + if err != nil { + return interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return interpretError(err) + } + + // MySQL only reports rows that actually changed values, not rows + // matched. If no rows were affected for a named queue, check if the + // queue exists before returning ErrNotFound (it may already be paused). + if rowsAffected < 1 && params.Name != riverdriver.AllQueuesString { + if _, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name); err != nil { + return interpretError(err) + } + } + return nil + }) +} + +func (e *Executor) QueueResume(ctx context.Context, params *riverdriver.QueueResumeParams) error { + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + now := nullTimeFromPtr(params.Now) + + var ( + res sql.Result + err error + ) + + if params.Name == riverdriver.AllQueuesString { + res, err = dbsqlc.New().QueueResumeAll(ctx, dbtx, now) + } else { + res, err = dbsqlc.New().QueueResumeByName(ctx, dbtx, &dbsqlc.QueueResumeByNameParams{ + Name: params.Name, + Now: now, + }) + } + if err != nil { + return interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return interpretError(err) + } + + // MySQL only reports rows that actually changed values, not rows + // matched. If no rows were affected for a named queue, check if the + // queue exists before returning ErrNotFound (it may already be resumed). + if rowsAffected < 1 && params.Name != riverdriver.AllQueuesString { + if _, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name); err != nil { + return interpretError(err) + } + } + return nil + }) +} + +func (e *Executor) QueueUpdate(ctx context.Context, params *riverdriver.QueueUpdateParams) (*rivertype.Queue, error) { + var metadataDoUpdate int64 + if params.MetadataDoUpdate { + metadataDoUpdate = 1 + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.Queue, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().QueueUpdateExec(ctx, dbtx, &dbsqlc.QueueUpdateExecParams{ + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + MetadataDoUpdate: metadataDoUpdate, + Name: params.Name, + }); err != nil { + return nil, interpretError(err) + } + + queue, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil + }) +} + +func (e *Executor) QueryRow(ctx context.Context, sql string, args ...any) riverdriver.Row { + return e.dbtx.QueryRowContext(ctx, sql, args...) +} + +func (e *Executor) SchemaCreate(ctx context.Context, params *riverdriver.SchemaCreateParams) error { + // In MySQL, schemas are databases. Create one if it doesn't exist. + if params.Schema != "" { + _, err := e.dbtx.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+mysqlIdentifier(params.Schema)) + return interpretError(err) + } + return nil +} + +func (e *Executor) SchemaDrop(ctx context.Context, params *riverdriver.SchemaDropParams) error { + if params.Schema != "" { + _, err := e.dbtx.ExecContext(ctx, "DROP DATABASE IF EXISTS "+mysqlIdentifier(params.Schema)) + return interpretError(err) + } + return nil +} + +func (e *Executor) SchemaGetExpired(ctx context.Context, params *riverdriver.SchemaGetExpiredParams) ([]string, error) { + schemas, err := dbsqlc.New().SchemaGetExpired(informationSchemaParam(ctx), e.dbtx, &dbsqlc.SchemaGetExpiredParams{ + BeforeName: params.BeforeName, + Prefix: params.Prefix, + }) + return schemas, interpretError(err) +} + +func (e *Executor) TableExists(ctx context.Context, params *riverdriver.TableExistsParams) (bool, error) { + ctx = informationSchemaParam(ctx) + + exists, err := dbsqlc.New().TableExists(ctx, e.dbtx, &dbsqlc.TableExistsParams{ + TableName: params.Table, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) TableTruncate(ctx context.Context, params *riverdriver.TableTruncateParams) error { + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + for _, table := range params.Table { + // MySQL's TRUNCATE TABLE is DDL and can't be used in transactions. + // Use DELETE FROM instead for transactional safety. + _, err := e.dbtx.ExecContext(ctx, "DELETE FROM "+maybeSchema+mysqlIdentifier(table)) + if err != nil { + return interpretError(err) + } + } + + return nil +} + +// executor preserves transaction identity when a method is invoked through an +// embedded Executor, so transaction helpers can reuse it without a savepoint. +func (e *Executor) executor() riverdriver.Executor { + if e.execTx != nil { + return e.execTx + } + return e +} + +type ExecutorTx struct { + Executor + + tx *sql.Tx +} + +func (t *ExecutorTx) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + executorSubTx := &ExecutorSubTx{ + beginOnce: &savepointutil.BeginOnlyOnce{}, + savepointNum: 0, + tx: t.tx, + } + executorSubTx.Executor = Executor{nil, templateReplaceWrapper{t.tx, &t.driver.replacer}, t.driver, executorSubTx} + return executorSubTx.Begin(ctx) +} + +func (t *ExecutorTx) Commit(ctx context.Context) error { + return t.tx.Commit() +} + +func (t *ExecutorTx) Rollback(ctx context.Context) error { + return t.tx.Rollback() +} + +type ExecutorSubTx struct { + Executor + + beginOnce *savepointutil.BeginOnlyOnce + savepointNum int + tx *sql.Tx +} + +const savepointPrefix = "river_savepoint_" + +func (t *ExecutorSubTx) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + if err := t.beginOnce.Begin(); err != nil { + return nil, err + } + + nextSavepointNum := t.savepointNum + 1 + if err := t.Exec(ctx, fmt.Sprintf("SAVEPOINT %s%02d", savepointPrefix, nextSavepointNum)); err != nil { + return nil, err + } + + executorSubTx := &ExecutorSubTx{ + beginOnce: savepointutil.NewBeginOnlyOnce(t.beginOnce), + savepointNum: nextSavepointNum, + tx: t.tx, + } + executorSubTx.Executor = Executor{nil, templateReplaceWrapper{t.tx, &t.driver.replacer}, t.driver, executorSubTx} + + return executorSubTx, nil +} + +func (t *ExecutorSubTx) Commit(ctx context.Context) error { + defer t.beginOnce.Done() + + if t.beginOnce.IsDone() { + return errors.New("tx is closed") + } + + if err := t.Exec(ctx, fmt.Sprintf("RELEASE SAVEPOINT %s%02d", savepointPrefix, t.savepointNum)); err != nil { + // MySQL DDL statements (CREATE TABLE, ALTER TABLE, etc.) cause an + // implicit COMMIT which destroys savepoints. If the savepoint no + // longer exists, the work was already committed by the DDL. + if isSavepointDoesNotExist(err) { + return nil + } + return err + } + + return nil +} + +func (t *ExecutorSubTx) Rollback(ctx context.Context) error { + defer t.beginOnce.Done() + + if t.beginOnce.IsDone() { + return errors.New("tx is closed") + } + + if err := t.Exec(ctx, fmt.Sprintf("ROLLBACK TO SAVEPOINT %s%02d", savepointPrefix, t.savepointNum)); err != nil { + // MySQL DDL statements cause implicit COMMIT which destroys + // savepoints. If the savepoint no longer exists, there's nothing + // to roll back to. + if isSavepointDoesNotExist(err) { + return nil + } + return err + } + + return nil +} + +// isSavepointDoesNotExist checks if a MySQL error indicates that a savepoint +// does not exist (error 1305). This happens when DDL statements cause an +// implicit COMMIT that destroys active savepoints. +func isSavepointDoesNotExist(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1305 +} + +func isDuplicateEntry(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 +} + +func boolToInt64(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func interpretError(err error) error { + if errors.Is(err, sql.ErrNoRows) { + return rivertype.ErrNotFound + } + return err +} + +// mysqlIdentifier quotes an identifier with backticks for MySQL. +// MySQL uses backticks instead of double quotes for identifier quoting. +func mysqlIdentifier(ident string) string { + return "`" + strings.ReplaceAll(ident, "`", "``") + "`" +} + +type templateReplaceWrapper struct { + dbtx dbsqlc.DBTX + replacer *sqlctemplate.Replacer +} + +func (w templateReplaceWrapper) ExecContext(ctx context.Context, rawSQL string, rawArgs ...any) (sql.Result, error) { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.ExecContext(ctx, sqlStr, args...) +} + +func (w templateReplaceWrapper) PrepareContext(ctx context.Context, rawSQL string) (*sql.Stmt, error) { + sqlStr, _ := w.replacer.Run(ctx, argPlaceholder, rawSQL, nil) + return w.dbtx.PrepareContext(ctx, sqlStr) +} + +func (w templateReplaceWrapper) QueryContext(ctx context.Context, rawSQL string, rawArgs ...any) (*sql.Rows, error) { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.QueryContext(ctx, sqlStr, args...) +} + +func (w templateReplaceWrapper) QueryRowContext(ctx context.Context, rawSQL string, rawArgs ...any) *sql.Row { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.QueryRowContext(ctx, sqlStr, args...) +} + +// jobGetAvailableResultFromInternal decodes the job rows locked by +// JobGetAvailable, retaining every row and recording decode errors by job ID +// because they've all been moved to `running`. +func jobGetAvailableResultFromInternal(jobs []*dbsqlc.RiverJob) *riverdriver.JobGetAvailableResult { + res := &riverdriver.JobGetAvailableResult{Jobs: make([]*rivertype.JobRow, len(jobs))} + for i, internal := range jobs { + job, err := jobRowFromInternalPartial(internal) + res.Jobs[i] = job + if err != nil { + if res.DecodeErrors == nil { + res.DecodeErrors = make(map[int64]error) + } + res.DecodeErrors[job.ID] = err + } + } + return res +} + +// jobRowFromInternal decodes a job row, returning an error if any of its +// fields can't be decoded. +func jobRowFromInternal(internal *dbsqlc.RiverJob) (*rivertype.JobRow, error) { + job, err := jobRowFromInternalPartial(internal) + if err != nil { + return nil, err + } + return job, nil +} + +func jobRowFromInternalPartial(internal *dbsqlc.RiverJob) (*rivertype.JobRow, error) { + attemptedAt := ptrFromNullTime(internal.AttemptedAt) + if attemptedAt != nil { + t := attemptedAt.UTC() + attemptedAt = &t + } + + var decodeErrs []error + var attemptedBy []string + if internal.AttemptedBy != nil { + if err := json.Unmarshal(internal.AttemptedBy, &attemptedBy); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `attempted_by`: %w", err)) + attemptedBy = nil + } + } + + errs := make([]rivertype.AttemptError, 0) + if internal.Errors != nil { + if err := riverdriver.UnmarshalAttemptErrors(internal.Errors, &errs); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `errors`: %w", err)) + errs = nil + } + } + + finalizedAt := ptrFromNullTime(internal.FinalizedAt) + if finalizedAt != nil { + t := finalizedAt.UTC() + finalizedAt = &t + } + + var tags []string + if err := json.Unmarshal(internal.Tags, &tags); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `tags`: %w", err)) + tags = nil + } + + var uniqueStatesByte byte + if internal.UniqueStates.Valid { + if internal.UniqueStates.Int16 < 0 || internal.UniqueStates.Int16 > 255 { + decodeErrs = append(decodeErrs, fmt.Errorf("value out of range for byte: %d", internal.UniqueStates.Int16)) + } else { + uniqueStatesByte = byte(internal.UniqueStates.Int16) + } + } + + return &rivertype.JobRow{ + ID: internal.ID, + Attempt: max(int(internal.Attempt), 0), + AttemptedAt: attemptedAt, + AttemptedBy: attemptedBy, + CreatedAt: internal.CreatedAt.UTC(), + EncodedArgs: internal.Args, + Errors: errs, + FinalizedAt: finalizedAt, + Kind: internal.Kind, + MaxAttempts: max(int(internal.MaxAttempts), 0), + Metadata: internal.Metadata, + Priority: max(int(internal.Priority), 0), + Queue: internal.Queue, + ScheduledAt: internal.ScheduledAt.UTC(), + State: rivertype.JobState(internal.State), + Tags: tags, + UniqueKey: bytesFromNullString(internal.UniqueKey), + UniqueStates: uniquestates.UniqueBitmaskToStates(uniqueStatesByte), + }, errors.Join(decodeErrs...) +} + +// jobRowsFromInternalPartial decodes job rows with jobRowFromInternalPartial, +// ignoring decode errors so that one bad row doesn't prevent returning the +// others. +func jobRowsFromInternalPartial(jobs []*dbsqlc.RiverJob) []*rivertype.JobRow { + return sliceutil.Map(jobs, func(internal *dbsqlc.RiverJob) *rivertype.JobRow { + job, _ := jobRowFromInternalPartial(internal) + return job + }) +} + +func leaderFromInternal(internal *dbsqlc.RiverLeader) *riverdriver.Leader { + return &riverdriver.Leader{ + ElectedAt: internal.ElectedAt.UTC(), + ExpiresAt: internal.ExpiresAt.UTC(), + LeaderID: internal.LeaderID, + } +} + +func migrationFromInternal(internal *dbsqlc.RiverMigration) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: internal.Line, + Version: int(internal.Version), + } +} + +func schemaTemplateParam(ctx context.Context, schema string) context.Context { + if schema != "" { + schema = mysqlIdentifier(schema) + "." + } + + return sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "schema": {Value: schema}, + }, nil) +} + +func informationSchemaParam(ctx context.Context) context.Context { + return sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "information_schema": {Stable: true, Value: "INFORMATION_SCHEMA."}, + }, nil) +} + +func queueFromInternal(internal *dbsqlc.RiverQueue) *rivertype.Queue { + pausedAt := ptrFromNullTime(internal.PausedAt) + if pausedAt != nil { + t := pausedAt.UTC() + pausedAt = &t + } + return &rivertype.Queue{ + CreatedAt: internal.CreatedAt.UTC(), + Metadata: internal.Metadata, + Name: internal.Name, + PausedAt: pausedAt, + UpdatedAt: internal.UpdatedAt.UTC(), + } +} + +func nullTimeFromPtr(t *time.Time) sql.NullTime { + if t == nil { + return sql.NullTime{} + } + return sql.NullTime{Time: t.UTC(), Valid: true} +} + +func ptrFromNullTime(t sql.NullTime) *time.Time { + if !t.Valid { + return nil + } + return &t.Time +} + +func nullStringFromBytes(b []byte) sql.NullString { + if b == nil { + return sql.NullString{} + } + return sql.NullString{String: string(b), Valid: true} +} + +func bytesFromNullString(s sql.NullString) []byte { + if !s.Valid { + return nil + } + return []byte(s.String) +} + +func utcTimePtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + return new(t.UTC()) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_batch.go b/riverdriver/rivermysql/river_mysql_driver_batch.go new file mode 100644 index 000000000..620bc89ea --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_batch.go @@ -0,0 +1,276 @@ +package rivermysql + +import ( + "context" + "database/sql" + _ "embed" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" + "github.com/riverqueue/river/rivershared/uniquestates" + "github.com/riverqueue/river/rivershared/util/randutil" + "github.com/riverqueue/river/rivershared/util/sliceutil" + "github.com/riverqueue/river/rivertype" +) + +// sqlc's MySQL parser doesn't support JSON_TABLE. Keep these batch queries +// outside its input directory and use the same schema replacement and executor +// as generated queries. Single-row operations continue to use sqlc. +var ( + //go:embed internal/dbbatch/job_insert.sql + jobInsertBatchSQL string + //go:embed internal/dbbatch/job_insert_upsert.sql + jobInsertBatchUpsertSQL string + //go:embed internal/dbbatch/job_rescue.sql + jobRescueBatchSQL string + //go:embed internal/dbbatch/job_set_state.sql + jobSetStateBatchSQL string + //go:embed internal/dbbatch/job_unique_key.sql + jobUniqueKeySQL string + //go:embed internal/dbbatch/notification_insert.sql + notificationInsertBatchSQL string +) + +// A joined UPDATE only updates each matching row once. Use the sequential path +// for repeated IDs to retain the ordering of state changes and metadata merges. +func batchIDsUnique(ids []int64) bool { + seen := make(map[int64]struct{}, len(ids)) + for _, id := range ids { + if _, ok := seen[id]; ok { + return false + } + seen[id] = struct{}{} + } + return true +} + +func batchTime(t *time.Time) *string { + if t == nil { + return nil + } + return new(t.UTC().Format("2006-01-02 15:04:05.000000")) +} + +func execJSONBatch(ctx context.Context, dbtx templateReplaceWrapper, query string, rows any, args ...any) error { + encoded, err := json.Marshal(rows) + if err != nil { + return err + } + _, err = dbtx.ExecContext(ctx, query, append([]any{encoded}, args...)...) + return interpretError(err) +} + +// MySQL requires a default database for joined updates containing derived +// tables, even with fully qualified table names. Retain the sequential path +// for connections without a default database; don't change pooled session state. +func isNoDatabaseSelected(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1046 +} + +func jobInsertBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobInsertFastManyParams, nonces []string) (sql.Result, error) { + // JSON documents travel as strings, then CAST back to JSON in SQL. This + // preserves JSON null (distinct from SQL NULL), large integers, and the + // database's validation of malformed JSON without decoding job arguments. + type row struct { + ID *int64 `json:"id"` + Args string `json:"args"` + CreatedAt *string `json:"created_at"` + Kind string `json:"kind"` + MaxAttempts int `json:"max_attempts"` + Metadata string `json:"metadata"` + Priority int `json:"priority"` + Queue string `json:"queue"` + ScheduledAt *string `json:"scheduled_at"` + State rivertype.JobState `json:"state"` + Tags string `json:"tags"` + UniqueActive bool `json:"unique_active"` + UniqueKey string `json:"unique_key"` + UniqueStates byte `json:"unique_states"` + } + rows := make([]row, len(params.Jobs)) + upsert := "" + for i, job := range params.Jobs { + tags, err := json.Marshal(job.Tags) + if err != nil { + return nil, err + } + metadata := sliceutil.FirstNonEmpty(job.Metadata, []byte("{}")) + if nonces != nil { + metadata, err = sjson.SetBytes(metadata, riverdriver.UniqueInsertMetadataKey, nonces[i]) + if err != nil { + return nil, err + } + } + rows[i] = row{ + ID: job.ID, + Args: string(job.EncodedArgs), + CreatedAt: batchTime(job.CreatedAt), + Kind: job.Kind, + MaxAttempts: job.MaxAttempts, + Metadata: string(metadata), + Priority: job.Priority, + Queue: job.Queue, + ScheduledAt: batchTime(job.ScheduledAt), + State: job.State, + Tags: string(tags), + UniqueActive: job.UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{job.State}) != 0, + UniqueKey: hex.EncodeToString(job.UniqueKey), + UniqueStates: job.UniqueStates, + } + if len(job.UniqueKey) > 0 { + upsert = jobInsertBatchUpsertSQL + } + } + encoded, err := json.Marshal(rows) + if err != nil { + return nil, err + } + ctx = sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Stable: true, Value: upsert}, + }, nil) + result, err := dbtx.ExecContext(ctx, jobInsertBatchSQL, encoded) + return result, interpretError(err) +} + +func jobInsertBatchReturning(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobInsertFastManyParams, nonces []string) ([]*dbsqlc.RiverJob, error) { + // Use the first generated ID only as a lower bound, never to infer the + // remaining IDs: concurrent INSERT ... SELECT statements may interleave them. + // A per-input nonce preserves order and distinguishes newly inserted jobs. + // The caller removes these temporary nonces before committing. + prefix := randutil.Hex(16) + ":" + for i := range nonces { + nonces[i] = prefix + strconv.Itoa(i) + } + result, err := jobInsertBatch(schemaTemplateParam(ctx, params.Schema), dbtx, params, nonces) + if err != nil { + return nil, err + } + firstID, err := result.LastInsertId() + if err != nil { + return nil, err + } + jobsByNonce := make(map[string]*dbsqlc.RiverJob, len(nonces)) + if firstID != 0 { + jobs, err := dbsqlc.New().JobGetByInsertNonce(schemaTemplateParam(ctx, params.Schema), dbtx, &dbsqlc.JobGetByInsertNonceParams{ + FirstID: firstID, NoncePrefix: []byte(prefix + "%"), + }) + if err != nil { + return nil, interpretError(err) + } + for _, job := range jobs { + jobsByNonce[gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).String()] = job + } + } + + inserted := make([]*dbsqlc.RiverJob, len(nonces)) + var missingKeys []sql.NullString + for i, nonce := range nonces { + inserted[i] = jobsByNonce[nonce] + if inserted[i] == nil && len(params.Jobs[i].UniqueKey) > 0 { + missingKeys = append(missingKeys, sql.NullString{String: string(params.Jobs[i].UniqueKey), Valid: true}) + } + } + if len(missingKeys) > 0 { + lookupCtx := sqlctemplate.WithReplacements(schemaTemplateParam(ctx, params.Schema), map[string]sqlctemplate.Replacement{ + "unique_key": {Stable: true, Value: "(" + jobUniqueKeySQL + ")"}, + }, nil) + jobs, err := dbsqlc.New().JobGetByUniqueKeyManyForUpdate(lookupCtx, dbtx, missingKeys) + if err != nil { + return nil, interpretError(err) + } + jobsByKey := make(map[string]*dbsqlc.RiverJob, len(jobs)) + for _, job := range jobs { + jobsByKey[job.UniqueKey.String] = job + } + for i, job := range params.Jobs { + if inserted[i] == nil { + inserted[i] = jobsByKey[string(job.UniqueKey)] + } + } + } + for i, job := range inserted { + if job == nil { + return nil, fmt.Errorf("could not find job inserted at index %d", i) + } + } + return inserted, nil +} + +func jobInsertBatchSupported(params *riverdriver.JobInsertFastManyParams) bool { + if len(params.Jobs) < 2 { + return false + } + for _, job := range params.Jobs { + // Explicit IDs may precede the first generated ID. Preserve non-object + // metadata and caller-supplied reserved keys through the sequential path. + if job.ID != nil || (len(job.Metadata) > 0 && (!json.Valid(job.Metadata) || !gjson.ParseBytes(job.Metadata).IsObject())) || + gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).Exists() { + return false + } + } + return true +} + +func jobRescueBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobRescueManyParams) error { + type row struct { + ID int64 `json:"id"` + Error *string `json:"error"` + FinalizedAt *string `json:"finalized_at"` + ScheduledAt *string `json:"scheduled_at"` + State string `json:"state"` + } + rows := make([]row, len(params.ID)) + for i, id := range params.ID { + rows[i] = row{ + ID: id, + FinalizedAt: batchTime(params.FinalizedAt[i]), + ScheduledAt: batchTime(¶ms.ScheduledAt[i]), + State: params.State[i], + } + if params.Error[i] != nil { + rows[i].Error = new(string(params.Error[i])) + } + } + return execJSONBatch(ctx, dbtx, jobRescueBatchSQL, rows, params.StuckHorizon.UTC()) +} + +func jobSetStateBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobSetStateIfRunningManyParams) error { + type row struct { + ID int64 `json:"id"` + Attempt *int `json:"attempt"` + Error *string `json:"error"` + FinalizedAt *string `json:"finalized_at"` + Metadata *string `json:"metadata"` + ScheduledAt *string `json:"scheduled_at"` + State rivertype.JobState `json:"state"` + } + rows := make([]row, len(params.ID)) + for i, id := range params.ID { + rows[i] = row{ + ID: id, + Attempt: params.Attempt[i], + FinalizedAt: batchTime(params.FinalizedAt[i]), + ScheduledAt: batchTime(params.ScheduledAt[i]), + State: params.State[i], + } + if params.ErrData[i] != nil { + rows[i].Error = new(string(params.ErrData[i])) + } + if params.MetadataDoMerge[i] { + rows[i].Metadata = new(string(params.MetadataUpdates[i])) + } + } + return execJSONBatch(ctx, dbtx, jobSetStateBatchSQL, rows, nullTimeFromPtr(params.Now)) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_batch_test.go b/riverdriver/rivermysql/river_mysql_driver_batch_test.go new file mode 100644 index 000000000..28314535b --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_batch_test.go @@ -0,0 +1,380 @@ +package rivermysql + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivertype" +) + +func TestJobInsertBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + exec riverdriver.Executor + params *riverdriver.JobInsertFastManyParams + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + params := &riverdriver.JobInsertFastManyParams{Schema: riverdbtest.TestSchema(ctx, t, driver, nil)} + for i := range 2 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + ID: new(int64(9007199254740993 + i)), EncodedArgs: []byte(`null`), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`null`), Priority: 1, + Queue: "default", State: rivertype.JobStateAvailable, + }) + } + return &testBundle{exec: driver.GetExecutor(), params: params} + } + + t.Run("JSONNullsAndIDs", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + bundle.params.Jobs[1].EncodedArgs = []byte(`{"id":9007199254740993}`) + bundle.params.Jobs[1].Metadata = []byte(`{"id":9007199254740993}`) + count, err := bundle.exec.JobInsertFastManyNoReturning(ctx, bundle.params) + require.NoError(t, err) + require.Equal(t, 2, count) + for i, params := range bundle.params.Jobs { + job, err := bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: *params.ID, Schema: bundle.params.Schema}) + require.NoError(t, err) + require.Equal(t, *params.ID, job.ID) + require.JSONEq(t, string(params.EncodedArgs), string(job.EncodedArgs)) + require.JSONEq(t, string(params.Metadata), string(job.Metadata)) + if i == 1 { + require.Contains(t, string(job.EncodedArgs), "9007199254740993") + require.Contains(t, string(job.Metadata), "9007199254740993") + } + require.Nil(t, job.Tags) + } + }) + + t.Run("RejectInvalidBatch", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + bundle.params.Jobs[1].EncodedArgs = []byte(`{`) + _, err = tx.JobInsertFastManyNoReturning(ctx, bundle.params) + require.Error(t, err) + count, err := tx.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) + require.Zero(t, count) + }) +} + +func TestJobInsertBatchReturning(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + driver *Driver + params *riverdriver.JobInsertFastManyParams + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + params := &riverdriver.JobInsertFastManyParams{Schema: riverdbtest.TestSchema(ctx, t, driver, nil)} + for i := range byte(2) { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`{"id":9007199254740993}`), + Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, + UniqueKey: []byte{0xff, 0, i}, UniqueStates: 0xff, + }) + } + return &testBundle{driver: driver, params: params} + } + + t.Run("AutoIncrementGaps", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = tx.ExecContext(context.Background(), "SET SESSION auto_increment_increment = DEFAULT") + _ = tx.Rollback() + }) + _, err = tx.ExecContext(ctx, "SET SESSION auto_increment_increment = 7") + require.NoError(t, err) + results, err := bundle.driver.UnwrapExecutor(tx).JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + require.Equal(t, int64(7), results[1].Job.ID-results[0].Job.ID) + for i, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(bundle.params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + require.JSONEq(t, string(bundle.params.Jobs[i].Metadata), string(result.Job.Metadata)) + stored, err := bundle.driver.UnwrapExecutor(tx).JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: result.Job.ID, Schema: bundle.params.Schema, + }) + require.NoError(t, err) + require.JSONEq(t, string(bundle.params.Jobs[i].Metadata), string(stored.Metadata)) + } + }) + + t.Run("ConcurrentBatches", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for worker := range 4 { + t.Run(fmt.Sprintf("Worker%d", worker), func(t *testing.T) { + t.Parallel() + + for iteration := range 10 { + params := &riverdriver.JobInsertFastManyParams{Schema: bundle.params.Schema} + for i := range 20 { + job := *bundle.params.Jobs[0] + job.EncodedArgs = fmt.Appendf(nil, `{"worker":%d,"iteration":%d,"index":%d}`, worker, iteration, i) + job.UniqueKey = nil + if i%2 == 0 { + job.UniqueKey = fmt.Appendf(nil, "%d/%d/%d", worker, iteration, i) + } + params.Jobs = append(params.Jobs, &job) + } + results, err := bundle.driver.GetExecutor().JobInsertFastMany(t.Context(), params) + require.NoError(t, err) + require.Len(t, results, len(params.Jobs)) + for i, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + } + } + }) + } + }) + + t.Run("CurrentRead", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) // Establish a snapshot before the duplicate exists. + seed, err := bundle.driver.GetExecutor().JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: bundle.params.Jobs[:1], Schema: bundle.params.Schema, + }) + require.NoError(t, err) + + results, err := tx.JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + require.True(t, results[0].UniqueSkippedAsDuplicate) + require.Equal(t, seed[0].Job.ID, results[0].Job.ID) + require.False(t, results[1].UniqueSkippedAsDuplicate) + }) + + t.Run("NonObjectAndReservedMetadata", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for _, metadata := range []string{`null`, `[1,2]`, `{"river:unique_nonce":"caller"}`} { + for _, job := range bundle.params.Jobs { + job.Metadata = []byte(metadata) + job.UniqueKey = nil + } + results, err := bundle.driver.GetExecutor().JobInsertFastMany(t.Context(), bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + for _, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, metadata, string(result.Job.Metadata)) + } + } + }) + + t.Run("ReadDoesNotLockOtherInserts", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for _, job := range bundle.params.Jobs { + job.UniqueKey = nil + } + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + dbtx := templateReplaceWrapper{dbtx: tx, replacer: &bundle.driver.replacer} + result, err := jobInsertBatch(schemaTemplateParam(ctx, bundle.params.Schema), dbtx, bundle.params, []string{"test:0", "test:1"}) + require.NoError(t, err) + firstID, err := result.LastInsertId() + require.NoError(t, err) + + otherTx, err := bundle.driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = otherTx.Rollback(ctx) }) + _, err = otherTx.JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + + // The primary-key range includes another session's uncommitted jobs. + // Locking the whole range here could deadlock concurrent insertions. + readCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + rows, err := dbsqlc.New().JobGetByInsertNonce(schemaTemplateParam(readCtx, bundle.params.Schema), dbtx, + &dbsqlc.JobGetByInsertNonceParams{FirstID: firstID, NoncePrefix: []byte("test:%")}) + require.NoError(t, err) + require.Len(t, rows, 2) + }) + + t.Run("UniqueLookupUsesIndex", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + _, err := bundle.driver.GetExecutor().JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + var plan string + err = bundle.driver.dbPool.QueryRowContext(ctx, + "EXPLAIN FORMAT=JSON SELECT id FROM "+mysqlIdentifier(bundle.params.Schema)+".river_job WHERE ("+jobUniqueKeySQL+") IN (?, ?)", + bundle.params.Jobs[0].UniqueKey, bundle.params.Jobs[1].UniqueKey).Scan(&plan) + require.NoError(t, err) + require.Contains(t, plan, `"key": "river_job_unique_idx"`) + }) + + t.Run("UnrelatedUniqueConflictIsAtomic", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + exec := bundle.driver.GetExecutor() + require.NoError(t, exec.Exec(ctx, "CREATE UNIQUE INDEX kind_unique ON "+mysqlIdentifier(bundle.params.Schema)+".river_job (kind)")) + t.Cleanup(func() { + require.NoError(t, exec.Exec(context.Background(), "DROP INDEX kind_unique ON "+mysqlIdentifier(bundle.params.Schema)+".river_job")) + }) + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: bundle.params.Jobs[:1], Schema: bundle.params.Schema}) + require.NoError(t, err) + bundle.params.Jobs[0].Kind = "new_kind" + bundle.params.Jobs[0].UniqueKey = []byte("new_key") + + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = tx.ExecContext(context.Background(), "SET SESSION sql_mode = DEFAULT") + _ = tx.Rollback() + }) + // Conflict rejection must not depend on strict SQL mode. + _, err = tx.ExecContext(ctx, "SET SESSION sql_mode = ''") + require.NoError(t, err) + txExec := bundle.driver.UnwrapExecutor(tx) + _, err = txExec.JobInsertFastMany(ctx, bundle.params) + require.ErrorContains(t, err, "Invalid JSON text") + count, err := txExec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) + require.Equal(t, 1, count) + }) +} + +func TestJobSetStateBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + driver *Driver + jobs []*rivertype.JobRow + params *riverdriver.JobSetStateIfRunningManyParams + schema string + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + schema := riverdbtest.TestSchema(ctx, t, driver, nil) + params := &riverdriver.JobSetStateIfRunningManyParams{Schema: schema} + jobs := make([]*rivertype.JobRow, 0, 2) + for range 2 { + job, err := driver.GetExecutor().JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + Attempt: 3, EncodedArgs: []byte(`{}`), Kind: "batch", MaxAttempts: 25, + Priority: 1, Queue: "default", Schema: schema, State: rivertype.JobStateRunning, Tags: []string{}, + }) + require.NoError(t, err) + jobs = append(jobs, job) + params.ID = append(params.ID, job.ID) + params.Attempt = append(params.Attempt, nil) + params.ErrData = append(params.ErrData, nil) + params.FinalizedAt = append(params.FinalizedAt, nil) + params.MetadataDoMerge = append(params.MetadataDoMerge, true) + params.MetadataUpdates = append(params.MetadataUpdates, []byte(`{"result":"merged"}`)) + params.ScheduledAt = append(params.ScheduledAt, new(time.Now().Add(time.Hour))) + params.State = append(params.State, rivertype.JobStateRetryable) + } + return &testBundle{driver: driver, jobs: jobs, params: params, schema: schema} + } + + t.Run("CurrentRead", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + // Ensure the joined batch query is used even if TEST_MYSQL_URL has no database. + _, err = tx.ExecContext(ctx, "USE "+mysqlIdentifier(bundle.schema)) + require.NoError(t, err) + exec := bundle.driver.UnwrapExecutor(tx) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: bundle.jobs[0].ID, Schema: bundle.schema}) + require.NoError(t, err) // Establish a REPEATABLE READ snapshot before the changes below. + + now := time.Now().UTC().Truncate(time.Microsecond) + _, err = bundle.driver.GetExecutor().JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: bundle.jobs[0].ID, CancelAttemptedAt: now, Now: &now, Schema: bundle.schema, + }) + require.NoError(t, err) + require.NoError(t, bundle.driver.GetExecutor().Exec(ctx, + "UPDATE "+mysqlIdentifier(bundle.schema)+".river_job SET state = 'completed', finalized_at = ? WHERE id = ?", now, bundle.jobs[1].ID)) + + rows, err := exec.JobSetStateIfRunningMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, rows, 2) + require.Equal(t, rivertype.JobStateCancelled, rows[0].State) + require.Equal(t, bundle.jobs[0].ScheduledAt, rows[0].ScheduledAt) + require.Equal(t, rivertype.JobStateCompleted, rows[1].State) + require.Equal(t, now, *rows[1].FinalizedAt) + require.JSONEq(t, `{"result":"merged"}`, string(rows[1].Metadata)) + }) + + t.Run("RepeatedIDs", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + bundle.params.ID[1] = bundle.params.ID[0] + bundle.params.State[1] = rivertype.JobStateScheduled + bundle.params.MetadataUpdates[0] = []byte(`{"first":true}`) + bundle.params.MetadataUpdates[1] = []byte(`{"second":true}`) + rows, err := bundle.driver.GetExecutor().JobSetStateIfRunningMany(t.Context(), bundle.params) + require.NoError(t, err) + require.Len(t, rows, 2) + for _, row := range rows { + require.Equal(t, rivertype.JobStateRetryable, row.State) + require.JSONEq(t, `{"first":true,"second":true}`, string(row.Metadata)) + } + }) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go b/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go new file mode 100644 index 000000000..9a4342693 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go @@ -0,0 +1,236 @@ +package rivermysql + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivertype" +) + +// BenchmarkJSONTableBatch includes a transaction and rollback in each iteration, +// so updates always act on running jobs and inserts don't grow the tables. +func BenchmarkJSONTableBatch(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + now := time.Now().UTC().Truncate(time.Microsecond) + attemptedAt := now.Add(-time.Hour) + + for _, operation := range []string{"Insert", "Notify", "Rescue", "SetState"} { + for _, size := range []int{1, 10, 100, 1000} { + b.Run(fmt.Sprintf("%s/%d", operation, size), func(b *testing.B) { + insertParams := &riverdriver.JobInsertFastManyParams{Schema: schema} + notifyParams := &riverdriver.NotifyManyParams{Schema: schema, Topic: "river_insert"} + rescueParams := &riverdriver.JobRescueManyParams{Schema: schema, StuckHorizon: now} + stateParams := &riverdriver.JobSetStateIfRunningManyParams{Schema: schema} + for range size { + insertParams.Jobs = append(insertParams.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"message":"benchmark","account_id":123456789}`), + Kind: "benchmark", + MaxAttempts: 25, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"benchmark"}, + }) + notifyParams.Payload = append(notifyParams.Payload, `{"queue":"default"}`) + if operation != "Rescue" && operation != "SetState" { + continue + } + jobs, err := exec.JobInsertFullMany(ctx, &riverdriver.JobInsertFullManyParams{ + Jobs: []*riverdriver.JobInsertFullParams{{ + Attempt: 1, AttemptedAt: &attemptedAt, EncodedArgs: []byte(`{}`), + Kind: "benchmark", MaxAttempts: 25, Priority: 1, Queue: "default", + State: rivertype.JobStateRunning, Tags: []string{}, + }}, + Schema: schema, + }) + require.NoError(b, err) + id := jobs[0].ID + rescueParams.ID = append(rescueParams.ID, id) + rescueParams.Error = append(rescueParams.Error, []byte(`{"attempt":1,"error":"stuck job"}`)) + rescueParams.FinalizedAt = append(rescueParams.FinalizedAt, nil) + rescueParams.ScheduledAt = append(rescueParams.ScheduledAt, now) + rescueParams.State = append(rescueParams.State, string(rivertype.JobStateAvailable)) + stateParams.ID = append(stateParams.ID, id) + stateParams.Attempt = append(stateParams.Attempt, nil) + stateParams.ErrData = append(stateParams.ErrData, nil) + stateParams.FinalizedAt = append(stateParams.FinalizedAt, &now) + stateParams.MetadataDoMerge = append(stateParams.MetadataDoMerge, true) + stateParams.MetadataUpdates = append(stateParams.MetadataUpdates, []byte(`{"result":"ok"}`)) + stateParams.ScheduledAt = append(stateParams.ScheduledAt, nil) + stateParams.State = append(stateParams.State, rivertype.JobStateCompleted) + } + + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + switch operation { + case "Insert": + count, err := tx.JobInsertFastManyNoReturning(ctx, insertParams) + require.NoError(b, err) + require.Equal(b, size, count) + case "Notify": + require.NoError(b, tx.NotifyMany(ctx, notifyParams)) + case "Rescue": + _, err := tx.JobRescueMany(ctx, rescueParams) + require.NoError(b, err) + case "SetState": + jobs, err := tx.JobSetStateIfRunningMany(ctx, stateParams) + require.NoError(b, err) + require.Len(b, jobs, size) + } + require.NoError(b, tx.Rollback(ctx)) + } + }) + } + } +} + +func BenchmarkJobGetAvailableKnownKinds(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + kinds := make([]string, 100) + for i := range kinds { + kinds[i] = fmt.Sprintf("kind_%03d", i) + } + jobs := make([]*riverdriver.JobInsertFastParams, 10_000) + for i := range jobs { + jobs[i] = &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{}`), + Kind: kinds[i%len(kinds)], + MaxAttempts: 1, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + } + } + _, err := exec.JobInsertFastManyNoReturning(ctx, &riverdriver.JobInsertFastManyParams{Jobs: jobs, Schema: schema}) + require.NoError(b, err) + + for _, testCase := range []struct { + kinds []string + name string + }{ + {kinds: kinds, name: "AllKnown100Kinds"}, + {kinds: nil, name: "Disabled"}, + {kinds: kinds[:1], name: "OnePercentKnown"}, + } { + b.Run(testCase.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + result, err := tx.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "benchmark", + Kind: testCase.kinds, + MaxAttemptedBy: 4, + MaxToLock: 100, + Queue: "default", + Schema: schema, + }) + require.NoError(b, err) + require.Len(b, result.Jobs, 100) + require.NoError(b, tx.Rollback(ctx)) + } + }) + } +} + +// BenchmarkJobInsertManyReturning measures the path used by standard InsertMany, +// including fetching returned jobs and classifying uniqueness conflicts. +func BenchmarkJobInsertManyReturning(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + for _, mode := range []string{"MixedDuplicates", "Plain", "Unique"} { + for _, size := range []int{1, 10, 100, 1000} { + b.Run(fmt.Sprintf("%s/%d", mode, size), func(b *testing.B) { + params := &riverdriver.JobInsertFastManyParams{Schema: schema} + for i := range size { + job := &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"message":"benchmark","index":%d}`, i)), + Kind: "benchmark", + MaxAttempts: 25, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"benchmark"}, + } + if mode != "Plain" { + job.UniqueKey = fmt.Appendf(nil, "%s/%d/%d", mode, size, i) + job.UniqueStates = 0xff + } + params.Jobs = append(params.Jobs, job) + if mode == "MixedDuplicates" && i%2 == 0 { + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: []*riverdriver.JobInsertFastParams{job}, Schema: schema}) + require.NoError(b, err) + } + } + + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + jobs, err := tx.JobInsertFastMany(ctx, params) + require.NoError(b, err) + require.Len(b, jobs, size) + require.NoError(b, tx.Rollback(ctx)) + } + }) + } + } +} + +func BenchmarkListenerNotificationDrain(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + pool := riversharedtest.DBPoolMySQL(ctx, b) + driver := New(pool) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + payloads := make([]string, 256) + for i := range payloads { + payloads[i] = strconv.Itoa(i) + } + require.NoError(b, driver.GetExecutor().NotifyMany(ctx, &riverdriver.NotifyManyParams{ + Payload: payloads, + Schema: schema, + Topic: "benchmark", + })) + + b.ReportAllocs() + for b.Loop() { + listener := &Listener{ + dbPool: pool, + isConnected: true, + replacer: &driver.replacer, + schema: schema, + topics: map[string]int64{"benchmark": 0}, + } + for _, payload := range payloads { + notification, err := listener.WaitForNotification(ctx) + require.NoError(b, err) + require.Equal(b, payload, notification.Payload) + } + } +} diff --git a/riverdriver/rivermysql/river_mysql_driver_json.go b/riverdriver/rivermysql/river_mysql_driver_json.go new file mode 100644 index 000000000..9e41b7759 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_json.go @@ -0,0 +1,136 @@ +package rivermysql + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + "text/template" + + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" +) + +// sqlc's MySQL parser does not support JSON_TABLE. Keep these predicate +// templates outside its input directory, alongside the other driver SQL. +// +//go:embed internal/dbfragment/json_contains.sql +var jsonContainsSQL string + +// Parsed once and used without mutation for all metadata filters. +var jsonContainsTemplates = template.Must(template.New("json_contains").Parse(jsonContainsSQL)) //nolint:gochecknoglobals + +// Merge under row locks before updating. JSON_MERGE_PATCH recursively merges +// objects and removes JSON nulls, while River replaces each supplied member. +// Read all metadata in one query to retain batching and preserve repeated IDs. +func jobMetadataMergeMany(ctx context.Context, dbtx templateReplaceWrapper, ids []int64, doMerge []bool, updates [][]byte) ([][]byte, error) { + var mergeIDs []int64 + for i, id := range ids { + if doMerge[i] { + mergeIDs = append(mergeIDs, id) + } + } + if len(mergeIDs) == 0 { + return updates, nil + } + rows, err := dbsqlc.New().JobGetMetadataForUpdate(ctx, dbtx, mergeIDs) + if err != nil { + return nil, err + } + metadata := make(map[int64]map[string]json.RawMessage, len(rows)) + for _, row := range rows { + value := make(map[string]json.RawMessage) + if err := json.Unmarshal(row.Metadata, &value); err != nil { + return nil, fmt.Errorf("decoding job %d metadata: %w", row.ID, err) + } + if value == nil { + value = make(map[string]json.RawMessage) + } + metadata[row.ID] = value + } + merged := slices.Clone(updates) + for i, id := range ids { + value, exists := metadata[id] + if !doMerge[i] || !exists { + continue + } + var update map[string]json.RawMessage + if err := json.Unmarshal(updates[i], &update); err != nil { + return nil, fmt.Errorf("decoding job %d metadata update: %w", id, err) + } + maps.Copy(value, update) + merged[i], err = json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encoding job %d metadata: %w", id, err) + } + } + return merged, nil +} + +// JSON_CONTAINS flattens nested arrays. Traverse each level explicitly to +// preserve Postgres containment boundaries, including whole object elements. +// Scalar subqueries avoid MySQL 8 semijoin rewrites losing nested correlations. +func jsonContains(source string, fragment json.RawMessage, bindFunc func(any) string, depth int) (string, error) { + fragment = bytes.TrimSpace(fragment) + values := map[string]string{"source": source} + query := "scalar" + switch fragment[0] { + case '{', '[': + alias := fmt.Sprintf("metadata_%d", depth) + var children []string + childFunc := func(query, param string, child json.RawMessage) error { + predicate, err := jsonContains(alias+".value", child, bindFunc, depth+1) + if err != nil { + return err + } + var sql strings.Builder + err = jsonContainsTemplates.ExecuteTemplate(&sql, query, map[string]string{ + "alias": alias, "param": param, "predicate": predicate, "source": source, + }) + children = append(children, sql.String()) + return err + } + if fragment[0] == '{' { + values["kind"] = "OBJECT" + var object map[string]json.RawMessage + if err := json.Unmarshal(fragment, &object); err != nil { + return "", err + } + for _, key := range slices.Sorted(maps.Keys(object)) { + encodedKey, err := json.Marshal(key) + if err != nil { + return "", err + } + if err := childFunc("member", bindFunc("$."+string(encodedKey)), object[key]); err != nil { + return "", err + } + } + } else { + values["kind"] = "ARRAY" + var array []json.RawMessage + if err := json.Unmarshal(fragment, &array); err != nil { + return "", err + } + for _, child := range array { + if err := childFunc("element", "", child); err != nil { + return "", err + } + } + } + values["children"] = "true" + if len(children) > 0 { + values["children"] = strings.Join(children, " AND ") + } + query = "container" + case 'n': + query = "null" + default: + values["param"] = bindFunc(string(fragment)) + } + var sql strings.Builder + err := jsonContainsTemplates.ExecuteTemplate(&sql, query, values) + return sql.String(), err +} diff --git a/riverdriver/rivermysql/river_mysql_driver_test.go b/riverdriver/rivermysql/river_mysql_driver_test.go new file mode 100644 index 000000000..e98d73007 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_test.go @@ -0,0 +1,1095 @@ +package rivermysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/testsignal" + "github.com/riverqueue/river/rivertype" +) + +// Verify interface compliance. +var _ riverdriver.Driver[*sql.Tx] = New(nil) + +func TestIsDuplicateEntry(t *testing.T) { + t.Parallel() + + require.True(t, isDuplicateEntry(&mysql.MySQLError{Number: 1062})) + require.False(t, isDuplicateEntry(&mysql.MySQLError{Number: 1305})) + require.False(t, isDuplicateEntry(errors.New("unrelated error containing 1062"))) +} + +func TestIsSavepointDoesNotExist(t *testing.T) { + t.Parallel() + + require.True(t, isSavepointDoesNotExist(&mysql.MySQLError{Number: 1305})) + require.False(t, isSavepointDoesNotExist(&mysql.MySQLError{Number: 1062})) + require.False(t, isSavepointDoesNotExist(errors.New("unrelated error containing 1305"))) +} + +func TestInterpretError(t *testing.T) { + t.Parallel() + + require.EqualError(t, interpretError(errors.New("an error")), "an error") + require.ErrorIs(t, interpretError(sql.ErrNoRows), rivertype.ErrNotFound) + require.NoError(t, interpretError(nil)) +} + +func TestSchemaTemplateParam(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("NoSchema", func(t *testing.T) { + t.Parallel() + ctx := schemaTemplateParam(ctx, "") + // Just verify it doesn't panic + _ = ctx + }) + + t.Run("WithSchema", func(t *testing.T) { + t.Parallel() + ctx := schemaTemplateParam(ctx, "custom_schema") + _ = ctx + }) +} + +func TestDriverProperties(t *testing.T) { + t.Parallel() + + driver := New(nil) + require.Equal(t, "?", driver.ArgPlaceholder()) + require.Equal(t, riverdriver.DatabaseNameMySQL, driver.DatabaseName()) + require.True(t, driver.SupportsListener()) + require.True(t, driver.SupportsListenNotify()) + require.Equal(t, time.Microsecond, driver.TimePrecision()) + require.False(t, driver.PoolIsSet()) +} + +func TestJobInsertFastManyUniqueConflictWithinBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + newJobParams := func() *riverdriver.JobInsertFastParams { + return &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"encoded": "args"}`), + Kind: "test_kind", + MaxAttempts: 25, + Metadata: []byte(`{"meta": "data"}`), + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"tag"}, + UniqueKey: []byte("unique-key-fast-conflict-within-batch"), + UniqueStates: 0xff, + } + } + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(), newJobParams()}, + Schema: schema, + }) + require.ErrorContains(t, err, "unique key appears more than once in batch") + require.Empty(t, results) +} + +func TestJobInsertFastManyUnrelatedUniqueConstraint(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + indexName = "river_job_test_kind_unique" + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_job") + ) + + require.NoError(t, exec.Exec(ctx, "CREATE UNIQUE INDEX "+mysqlIdentifier(indexName)+" ON "+table+" (kind)")) + t.Cleanup(func() { + require.NoError(t, exec.Exec(context.Background(), "DROP INDEX "+mysqlIdentifier(indexName)+" ON "+table)) + }) + + newJobParams := func(uniqueStates byte) *riverdriver.JobInsertFastParams { + return &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"encoded": "args"}`), + Kind: "test_kind", + MaxAttempts: 25, + Metadata: []byte(`{"meta": "data"}`), + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"tag"}, + UniqueKey: []byte("same-unique-key"), + UniqueStates: uniqueStates, + } + } + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(0xff)}, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 1) + + _, err = exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + // The requested available job's River uniqueness constraint is inactive, + // so matching the existing key must not hide the unrelated kind conflict. + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(1 << 2)}, + Schema: schema, + }) + require.EqualError(t, err, "job insert at index 0 conflicted on a non-River unique constraint") + + jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{Kind: []string{"test_kind"}, Schema: schema}) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, []byte("same-unique-key"), jobs[0].UniqueKey) +} + +func TestJobInsertAndGet(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{"test": true}`), + Kind: "test_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{"tag1"}, + }) + require.NoError(t, err) + require.NotNil(t, job) + require.Positive(t, job.ID) + require.Equal(t, "test_job", job.Kind) + require.Equal(t, rivertype.JobStateAvailable, job.State) + require.Equal(t, "default", job.Queue) + require.Equal(t, 1, job.Priority) + require.Equal(t, 3, job.MaxAttempts) + require.JSONEq(t, `{"test": true}`, string(job.EncodedArgs)) + require.Equal(t, []string{"tag1"}, job.Tags) + + // Get the job by ID + fetched, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: job.ID, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, job.ID, fetched.ID) + require.Equal(t, job.Kind, fetched.Kind) +} + +func TestJobGetAvailable(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert some available jobs + for i := range 3 { + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: fmt.Sprintf("job_%d", i), + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + } + + // Get available jobs (needs transaction for FOR UPDATE) + txExec, err := exec.Begin(ctx) + require.NoError(t, err) + defer txExec.Rollback(ctx) + + jobs, err := txExec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "test-client", + MaxAttemptedBy: 4, + MaxToLock: 2, + Queue: "default", + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, jobs.Jobs, 2) + + for _, job := range jobs.Jobs { + require.Equal(t, rivertype.JobStateRunning, job.State) + } + + require.NoError(t, txExec.Commit(ctx)) +} + +func TestJobGetAvailableConcurrentClaimsAreUnique(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + const ( + numJobs = 50 + numWorkers = 10 + ) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + jobs := make([]*riverdriver.JobInsertFastParams, numJobs) + for i := range jobs { + jobs[i] = &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{}`), + Kind: "concurrent_claim", + MaxAttempts: 3, + Priority: 1, + Queue: "concurrent_claims", + State: rivertype.JobStateAvailable, + Tags: []string{}, + } + } + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: jobs, Schema: schema}) + require.NoError(t, err) + + var ( + claimedIDs []int64 + errs []error + mu sync.Mutex + ready sync.WaitGroup + start = make(chan struct{}) + workers sync.WaitGroup + ) + ready.Add(numWorkers) + workers.Add(numWorkers) + for range numWorkers { + go func() { + defer workers.Done() + ready.Done() + <-start + + claimed, err := exec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "concurrent-client", + MaxAttemptedBy: 4, + MaxToLock: numJobs / numWorkers, + Queue: "concurrent_claims", + Schema: schema, + }) + + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, err) + return + } + for _, job := range claimed.Jobs { + claimedIDs = append(claimedIDs, job.ID) + } + }() + } + + ready.Wait() + close(start) + workers.Wait() + + require.NoError(t, errors.Join(errs...)) + require.Len(t, claimedIDs, numJobs) + uniqueClaimedIDs := make(map[int64]struct{}, len(claimedIDs)) + for _, id := range claimedIDs { + uniqueClaimedIDs[id] = struct{}{} + } + require.Len(t, uniqueClaimedIDs, numJobs) +} + +func TestJobDeleteWaitsForConcurrentStateChange(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "delete_concurrent_state_change", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + tx, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + + _, err = tx.ExecContext(ctx, "UPDATE "+mysqlIdentifier(schema)+".river_job SET state = 'running' WHERE id = ?", job.ID) //nolint:gosec + require.NoError(t, err) + + type deleteResult struct { + err error + job *rivertype.JobRow + } + deleteFinished := testsignal.TestSignal[deleteResult]{} + deleteFinished.Init(t) + deleteStarted := testsignal.TestSignal[struct{}]{} + deleteStarted.Init(t) + + go func() { + deleteStarted.Signal(struct{}{}) + job, err := exec.JobDelete(ctx, &riverdriver.JobDeleteParams{ID: job.ID, Schema: schema}) + deleteFinished.Signal(deleteResult{err: err, job: job}) + }() + + deleteStarted.WaitOrTimeout() + select { + case <-deleteFinished.WaitC(): + require.FailNow(t, "JobDelete returned while the concurrent update was uncommitted") + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, tx.Commit()) + + result := deleteFinished.WaitOrTimeout() + require.ErrorIs(t, result.err, rivertype.ErrJobRunning) + require.Nil(t, result.job) +} + +func TestJobCancel(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "test_cancel", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + // Cancel the job + cancelled, err := exec.JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: job.ID, + CancelAttemptedAt: time.Now().UTC(), + ControlTopic: "test_topic", + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateCancelled, cancelled.State) + require.NotNil(t, cancelled.FinalizedAt) +} + +func TestJobDelete(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a completed job + now := time.Now().UTC().Truncate(time.Microsecond) + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + FinalizedAt: &now, + Kind: "test_delete", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateCompleted, + Tags: []string{}, + }) + require.NoError(t, err) + + // Delete the job + deleted, err := exec.JobDelete(ctx, &riverdriver.JobDeleteParams{ + ID: job.ID, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, job.ID, deleted.ID) + + // Verify it's gone + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: job.ID, + Schema: schema, + }) + require.ErrorIs(t, err, rivertype.ErrNotFound) +} + +func TestJobCountByState(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert jobs in different states + for range 3 { + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "test_count", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + } + + count, err := exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 3, count) +} + +func TestLeaderElection(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Attempt to elect a leader + leader, err := exec.LeaderAttemptElect(ctx, &riverdriver.LeaderElectParams{ + LeaderID: "test-leader", + Schema: schema, + TTL: 30 * time.Second, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", leader.LeaderID) + + // Get the elected leader + fetched, err := exec.LeaderGetElectedLeader(ctx, &riverdriver.LeaderGetElectedLeaderParams{ + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", fetched.LeaderID) + + // Re-elect should succeed + reelected, err := exec.LeaderAttemptReelect(ctx, &riverdriver.LeaderReelectParams{ + ElectedAt: leader.ElectedAt, + LeaderID: "test-leader", + Schema: schema, + TTL: 30 * time.Second, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", reelected.LeaderID) + + // Resign + resigned, err := exec.LeaderResign(ctx, &riverdriver.LeaderResignParams{ + ElectedAt: leader.ElectedAt, + LeaderID: "test-leader", + LeadershipTopic: "leadership", + Schema: schema, + }) + require.NoError(t, err) + require.True(t, resigned) +} + +func TestListenerConnectDoesNotSkipLowerUncommittedID(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: schema}).(*Listener) //nolint:forcetypeassert + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_notification") + ) + insertNotificationSQL := "INSERT INTO " + table + " (payload, topic) VALUES (?, ?)" //nolint:gosec + + txLower, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = txLower.Rollback() }) + + _, err = txLower.ExecContext(ctx, insertNotificationSQL, "lower", "topic") + require.NoError(t, err) + + insertCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + higherResult, err := dbPool.ExecContext(insertCtx, insertNotificationSQL, "higher", "topic") + require.NoError(t, err) + higherID, err := higherResult.LastInsertId() + require.NoError(t, err) + + connectFinished := testsignal.TestSignal[error]{} + connectFinished.Init(t) + connectStarted := testsignal.TestSignal[struct{}]{} + connectStarted.Init(t) + + go func() { + connectStarted.Signal(struct{}{}) + connectFinished.Signal(listener.Connect(ctx)) + }() + + connectStarted.WaitOrTimeout() + select { + case err := <-connectFinished.WaitC(): + require.NoError(t, err) + require.FailNow(t, "listener connected before the lower transaction committed") + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, txLower.Commit()) + require.NoError(t, connectFinished.WaitOrTimeout()) + require.Equal(t, higherID, listener.lastID) +} + +func TestListenerWaitForNotificationDoesNotSkipLowerUncommittedID(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type waitForNotificationResult struct { + err error + found bool + notification *riverdriver.Notification + } + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: schema}).(*Listener) //nolint:forcetypeassert + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_notification") + ) + insertNotificationSQL := "INSERT INTO " + table + " (payload, topic) VALUES (?, ?)" //nolint:gosec + + listener.pollInterval = time.Millisecond + require.NoError(t, listener.Connect(ctx)) + t.Cleanup(func() { require.NoError(t, listener.Close(context.Background())) }) + require.NoError(t, listener.Listen(ctx, "topic")) + + txLower, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = txLower.Rollback() }) + + _, err = txLower.ExecContext(ctx, insertNotificationSQL, "lower", "topic") + require.NoError(t, err) + + insertCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + _, err = dbPool.ExecContext(insertCtx, insertNotificationSQL, "higher", "topic") + require.NoError(t, err) + + started := testsignal.TestSignal[struct{}]{} + started.Init(t) + waitForNotification := testsignal.TestSignal[waitForNotificationResult]{} + waitForNotification.Init(t) + + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + started.Signal(struct{}{}) + + notification, found, err := listener.waitForNotificationOnce(waitCtx) + waitForNotification.Signal(waitForNotificationResult{ + err: err, + found: found, + notification: notification, + }) + }() + + started.WaitOrTimeout() + + select { + case res := <-waitForNotification.WaitC(): + require.NoError(t, res.err) + require.FailNow(t, "listener returned before lower transaction committed", "notification: %+v", res.notification) + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, txLower.Commit()) + + res := waitForNotification.WaitOrTimeout() + require.NoError(t, res.err) + require.True(t, res.found) + require.Equal(t, &riverdriver.Notification{Payload: "lower", Topic: "topic"}, res.notification) + + waitHigherCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + notification, err := listener.WaitForNotification(waitHigherCtx) + require.NoError(t, err) + require.Equal(t, &riverdriver.Notification{Payload: "higher", Topic: "topic"}, notification) +} + +func TestQueueOperations(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + now := time.Now().UTC().Truncate(time.Microsecond) + + // Create a queue + queue, err := exec.QueueCreateOrSetUpdatedAt(ctx, &riverdriver.QueueCreateOrSetUpdatedAtParams{ + Metadata: []byte(`{}`), + Name: "test_queue", + Now: &now, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test_queue", queue.Name) + require.Nil(t, queue.PausedAt) + + // Get the queue + fetched, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test_queue", fetched.Name) + + // List queues + queues, err := exec.QueueList(ctx, &riverdriver.QueueListParams{ + Max: 100, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, queues, 1) + + // Pause queue + err = exec.QueuePause(ctx, &riverdriver.QueuePauseParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + + paused, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.NotNil(t, paused.PausedAt) + + // Resume queue + err = exec.QueueResume(ctx, &riverdriver.QueueResumeParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + + resumed, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.Nil(t, resumed.PausedAt) +} + +func TestServerTimestampsAreUTC(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + execTx, err := driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if closed { + return + } + _ = execTx.Exec(context.Background(), "SET time_zone = '+00:00'") + _ = execTx.Rollback(context.Background()) + }) + + require.NoError(t, execTx.Exec(ctx, "SET time_zone = '-05:00'")) + + beforeInsert := time.Now().UTC().Add(-time.Second) + job, err := execTx.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "server_timestamp_utc", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + afterInsert := time.Now().UTC().Add(time.Second) + + require.WithinRange(t, job.CreatedAt, beforeInsert, afterInsert) + require.WithinRange(t, job.ScheduledAt, beforeInsert, afterInsert) + + require.NoError(t, execTx.Exec(ctx, "SET time_zone = '+00:00'")) + require.NoError(t, execTx.Rollback(ctx)) + closed = true +} + +func TestTimeParametersAreNormalizedToUTC(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + localTime := time.Date(2026, time.July, 24, 12, 34, 56, 123456000, time.FixedZone("UTC-7", -7*60*60)) + job, err := driver.GetExecutor().JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + CreatedAt: &localTime, + EncodedArgs: []byte(`{}`), + Kind: "time_parameters_utc", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + ScheduledAt: &localTime, + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + require.Equal(t, localTime.UTC(), job.CreatedAt) + require.Equal(t, localTime.UTC(), job.ScheduledAt) +} + +func TestTransactions(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Begin and commit + tx, err := exec.Begin(ctx) + require.NoError(t, err) + + _, err = tx.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "tx_test", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + require.NoError(t, tx.Commit(ctx)) + + // Verify job exists + count, err := exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 1, count) + + // Begin and rollback + tx2, err := exec.Begin(ctx) + require.NoError(t, err) + + _, err = tx2.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "tx_test_rollback", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + require.NoError(t, tx2.Rollback(ctx)) + + // Verify second job was rolled back + count, err = exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 1, count) // still 1 +} + +func TestJobInsertFastMany(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{ + { + EncodedArgs: []byte(`{"i": 1}`), + Kind: "fast_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + { + EncodedArgs: []byte(`{"i": 2}`), + Kind: "fast_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + }, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 2) + + for _, result := range results { + require.NotNil(t, result.Job) + require.False(t, result.UniqueSkippedAsDuplicate) + } +} + +func TestJobMetadata(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job with metadata + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "metadata_test", + MaxAttempts: 3, + Metadata: []byte(`{"key": "value"}`), + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + var metadata map[string]any + require.NoError(t, json.Unmarshal(job.Metadata, &metadata)) + require.Equal(t, "value", metadata["key"]) + + // Update metadata + updated, err := exec.JobUpdate(ctx, &riverdriver.JobUpdateParams{ + ID: job.ID, + MetadataDoMerge: true, + Metadata: []byte(`{"new_key": "new_value"}`), + Schema: schema, + }) + require.NoError(t, err) + + require.NoError(t, json.Unmarshal(updated.Metadata, &metadata)) + require.Equal(t, "value", metadata["key"]) + require.Equal(t, "new_value", metadata["new_key"]) +} + +func TestJobSchedule(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a scheduled job with a past scheduled_at + past := time.Now().UTC().Add(-1 * time.Hour).Truncate(time.Microsecond) + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "scheduled_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + ScheduledAt: &past, + Schema: schema, + State: rivertype.JobStateScheduled, + Tags: []string{}, + }) + require.NoError(t, err) + + // Schedule should find and transition the job + now := time.Now().UTC() + results, err := exec.JobSchedule(ctx, &riverdriver.JobScheduleParams{ + Max: 100, + Now: &now, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, rivertype.JobStateAvailable, results[0].Job.State) +} + +func TestNotifyMany(t *testing.T) { + t.Parallel() + + driver := New(nil) + require.NotNil(t, driver.GetListener(&riverdriver.GetListenenerParams{})) +} + +func TestNotifyManyInsertsNotifications(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + exec = driver.GetExecutor() + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + payloads := []string{"test1", "", "é🌊\n\"quoted\"", "null", "test2"} + err := exec.NotifyMany(ctx, &riverdriver.NotifyManyParams{ + Payload: payloads, + Schema: schema, + Topic: "test_topic", + }) + require.NoError(t, err) + + rows, err := driver.dbPool.QueryContext(ctx, "SELECT payload, topic FROM "+mysqlIdentifier(schema)+".river_notification ORDER BY id") //nolint:gosec + require.NoError(t, err) + defer rows.Close() + var received []string + for rows.Next() { + var payload, topic string + require.NoError(t, rows.Scan(&payload, &topic)) + require.Equal(t, "test_topic", topic) + received = append(received, payload) + } + require.NoError(t, rows.Err()) + require.Equal(t, payloads, received) +} + +func TestPGAdvisoryXactLock(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + exec = New(riversharedtest.DBPoolMySQL(ctx, t)).GetExecutor() + ) + + _, err := exec.PGAdvisoryXactLock(ctx, 12345) + require.ErrorIs(t, err, riverdriver.ErrNotImplemented) +} diff --git a/riverdriver/rivermysql/river_mysql_listener.go b/riverdriver/rivermysql/river_mysql_listener.go new file mode 100644 index 000000000..cd48c6103 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_listener.go @@ -0,0 +1,316 @@ +package rivermysql + +import ( + "context" + "database/sql" + "errors" + "sync" + "time" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" +) + +const ( + notificationBatchSize = 256 + notificationPollIntervalDefault = 50 * time.Millisecond +) + +// Listener receives MySQL notifications from the river_notification outbox +// table. MySQL doesn't have a native LISTEN/NOTIFY equivalent, so NotifyMany +// appends rows to river_notification and this listener polls for rows with IDs +// greater than its remembered lastID. Polls use a locking read because InnoDB +// may commit auto-increment IDs out of order across concurrent transactions; a +// normal consistent read could otherwise skip a lower uncommitted notification. +type Listener struct { + afterConnectExec string // should only ever be used in testing + dbPool *sql.DB + isConnected bool + lastID int64 + mu sync.Mutex + pending []*dbsqlc.RiverNotification + pollInterval time.Duration + replacer *sqlctemplate.Replacer + schema string + topics map[string]int64 +} + +func (l *Listener) Close(context.Context) error { + l.mu.Lock() + defer l.mu.Unlock() + + l.isConnected = false + l.pending = nil + l.topics = nil + return nil +} + +func (l *Listener) Connect(ctx context.Context) error { + var ( + afterConnectExec string + dbPool *sql.DB + replacer *sqlctemplate.Replacer + schema string + ) + + l.mu.Lock() + if l.isConnected { + l.mu.Unlock() + return errors.New("connection already established") + } + afterConnectExec = l.afterConnectExec + dbPool = l.dbPool + replacer = l.replacer + schema = l.schema + l.mu.Unlock() + + if dbPool == nil { + return errors.New("database pool is nil") + } + if replacer == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } + + if afterConnectExec != "" { + if _, err := dbPool.ExecContext(ctx, afterConnectExec); err != nil { + return err + } + } + + // Establish the initial high-water mark with a locking scan. InnoDB allocates + // auto-increment IDs before commit, so a higher ID can commit while a lower + // ID is still uncommitted. A normal MAX(id) read would advance past the lower + // row and permanently skip it when it commits. Scan the IDs in ascending + // order with FOR UPDATE so the read reaches and waits for every lower + // uncommitted row before establishing the high-water mark. In particular, + // this must not be reduced to an aggregate MAX or descending LIMIT query, + // neither of which has to encounter the lower row. + tx, err := dbPool.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + ids, err := dbsqlc.New().NotificationGetIDsForUpdate(schemaTemplateParam(ctx, schema), notificationDBTX(tx, replacer)) + if err != nil { + return err + } + var lastID int64 + if len(ids) > 0 { + lastID = ids[len(ids)-1] + } + if err := tx.Commit(); err != nil { + return err + } + + l.mu.Lock() + defer l.mu.Unlock() + + if l.isConnected { + return errors.New("connection already established") + } + + l.isConnected = true + l.lastID = lastID + + return nil +} + +func (l *Listener) Listen(ctx context.Context, topic string) error { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return errors.New("listener is not connected") + } + + if _, ok := l.topics[topic]; ok { + return nil + } + + // Establish a starting ID for this topic without skipping pending messages + // for existing subscriptions. As in Connect, scan in ascending order with + // locks to account for notifications that haven't committed yet. + tx, err := l.dbPool.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + ids, err := dbsqlc.New().NotificationGetIDsForUpdate(schemaTemplateParam(ctx, l.schema), notificationDBTX(tx, l.replacer)) + if err != nil { + return err + } + lastID := l.lastID + if len(ids) > 0 { + lastID = max(lastID, ids[len(ids)-1]) + } + if err := tx.Commit(); err != nil { + return err + } + + if l.topics == nil { + l.topics = make(map[string]int64) + } + + l.topics[topic] = lastID + return nil +} + +func (l *Listener) Ping(ctx context.Context) error { + dbPool, err := l.stateDBPool() + if err != nil { + return err + } + return dbPool.PingContext(ctx) +} + +func (l *Listener) Schema() string { + l.mu.Lock() + defer l.mu.Unlock() + + return l.schema +} + +func (l *Listener) SetAfterConnectExec(sql string) { + l.mu.Lock() + defer l.mu.Unlock() + + l.afterConnectExec = sql +} + +func (l *Listener) Unlisten(_ context.Context, topic string) error { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return errors.New("listener is not connected") + } + + delete(l.topics, topic) + return nil +} + +func (l *Listener) WaitForNotification(ctx context.Context) (*riverdriver.Notification, error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + + notification, found, err := l.waitForNotificationOnce(ctx) + if errors.Is(err, sql.ErrNoRows) { + if err := l.waitForNextPoll(ctx); err != nil { + return nil, err + } + continue + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, err + } + if found { + return notification, nil + } + } +} + +func (l *Listener) stateDBPool() (*sql.DB, error) { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return nil, errors.New("listener is not connected") + } + if l.dbPool == nil { + return nil, errors.New("database pool is nil") + } + + return l.dbPool, nil +} + +func (l *Listener) waitForNextPoll(ctx context.Context) error { + l.mu.Lock() + pollInterval := l.pollInterval + l.mu.Unlock() + + if pollInterval <= 0 { + pollInterval = notificationPollIntervalDefault + } + + timer := time.NewTimer(pollInterval) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (l *Listener) waitForNotificationOnce(ctx context.Context) (*riverdriver.Notification, bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return nil, false, errors.New("listener is not connected") + } + if l.dbPool == nil { + return nil, false, errors.New("database pool is nil") + } + + if len(l.pending) == 0 { + if len(l.topics) == 0 { + return nil, false, sql.ErrNoRows + } + tx, err := l.dbPool.BeginTx(ctx, nil) + if err != nil { + return nil, false, err + } + defer tx.Rollback() + + // Preserve ascending locking reads: a lower ID might still be + // uncommitted. Shared locks allow other listeners to poll concurrently. + // Read all topics so this scan follows the primary key's order. + notifications, err := dbsqlc.New().NotificationGetAfter( + schemaTemplateParam(ctx, l.schema), notificationDBTX(tx, l.replacer), + &dbsqlc.NotificationGetAfterParams{After: l.lastID, Limit: notificationBatchSize}, + ) + if err != nil { + return nil, false, err + } + if err := tx.Commit(); err != nil { + return nil, false, err + } + if len(notifications) == 0 { + return nil, false, sql.ErrNoRows + } + l.pending = notifications + l.lastID = notifications[len(notifications)-1].ID + } + + for len(l.pending) > 0 { + notification := l.pending[0] + l.pending[0] = nil + l.pending = l.pending[1:] + + // Subscriptions may change after fetching a batch but before delivery. + if startID, ok := l.topics[notification.Topic]; ok && notification.ID > startID { + return &riverdriver.Notification{ + Payload: notification.Payload, + Topic: notification.Topic, + }, true, nil + } + } + return nil, false, nil +} + +func notificationDBTX(dbtx dbsqlc.DBTX, replacer *sqlctemplate.Replacer) templateReplaceWrapper { + if replacer == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } + return templateReplaceWrapper{dbtx, replacer} +} diff --git a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql index d87361114..59d4f988c 100644 --- a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql +++ b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql @@ -648,7 +648,9 @@ WITH job_input AS ( unnest(@finalized_at_do_update::boolean[]) AS finalized_at_do_update, unnest(@finalized_at::timestamptz[]) AS finalized_at, unnest(@metadata_do_merge::boolean[]) AS metadata_do_merge, - unnest(@metadata_updates::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest(@metadata_updates::text[])::jsonb AS metadata_updates, unnest(@scheduled_at_do_update::boolean[]) AS scheduled_at_do_update, unnest(@scheduled_at::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] diff --git a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go index a58f06ff0..b0007b617 100644 --- a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go +++ b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go @@ -1482,7 +1482,9 @@ WITH job_input AS ( unnest($6::boolean[]) AS finalized_at_do_update, unnest($7::timestamptz[]) AS finalized_at, unnest($8::boolean[]) AS metadata_do_merge, - unnest($9::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest($9::text[])::jsonb AS metadata_updates, unnest($10::boolean[]) AS scheduled_at_do_update, unnest($11::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] @@ -1562,7 +1564,7 @@ type JobSetStateIfRunningManyParams struct { FinalizedAtDoUpdate []bool FinalizedAt []time.Time MetadataDoMerge []bool - MetadataUpdates [][]byte + MetadataUpdates []string ScheduledAtDoUpdate []bool ScheduledAt []time.Time State []string diff --git a/riverdriver/riverpgxv5/river_pgx_v5_driver.go b/riverdriver/riverpgxv5/river_pgx_v5_driver.go index 71d56fef6..572efaf6d 100644 --- a/riverdriver/riverpgxv5/river_pgx_v5_driver.go +++ b/riverdriver/riverpgxv5/river_pgx_v5_driver.go @@ -65,8 +65,9 @@ func New(dbPool *pgxpool.Pool) *Driver { const argPlaceholder = "$" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{templateReplaceWrapper{d.dbPool, &d.replacer}, d} @@ -103,6 +104,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] return fmt.Sprintf("%s && @%s", column, namedArg), values, nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return column + " @> @" + namedArg + "::jsonb", map[string]any{namedArg: fragment}, nil +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { return fmt.Sprintf("%s = any(@%s)", column, column), values, nil } @@ -687,7 +692,7 @@ func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdr FinalizedAt: make([]time.Time, len(params.ID)), FinalizedAtDoUpdate: make([]bool, len(params.ID)), MetadataDoMerge: make([]bool, len(params.ID)), - MetadataUpdates: make([][]byte, len(params.ID)), + MetadataUpdates: make([]string, len(params.ID)), Now: params.Now, ScheduledAt: make([]time.Time, len(params.ID)), ScheduledAtDoUpdate: make([]bool, len(params.ID)), @@ -708,7 +713,9 @@ func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdr } if params.MetadataDoMerge[i] { setStateParams.MetadataDoMerge[i] = true - setStateParams.MetadataUpdates[i] = params.MetadataUpdates[i] + setStateParams.MetadataUpdates[i] = string(params.MetadataUpdates[i]) + } else { + setStateParams.MetadataUpdates[i] = "{}" } if params.ScheduledAt[i] != nil { setStateParams.ScheduledAtDoUpdate[i] = true diff --git a/riverdriver/riversqlite/internal/dbsqlc/river_job.sql b/riverdriver/riversqlite/internal/dbsqlc/river_job.sql index 6e651cbfe..3f800cdc6 100644 --- a/riverdriver/riversqlite/internal/dbsqlc/river_job.sql +++ b/riverdriver/riversqlite/internal/dbsqlc/river_job.sql @@ -696,10 +696,22 @@ RETURNING -- for JobSetStateIfRunning to use when falling back to non-running jobs. -- Metadata that isn't valid JSON is left in place, and columns are listed out -- to tolerate values that aren't valid JSON. See JobGetAvailable. +-- Rebuild top-level members rather than using JSON Merge Patch, which removes +-- JSON nulls and recursively merges nested objects. -- name: JobSetMetadataIfNotRunning :one UPDATE /* TEMPLATE: schema */river_job SET metadata = CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) - THEN jsonb_patch(json(metadata), json(@metadata_updates)) + THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(@metadata_updates, '$.' || json_quote(member_key)) IS NOT NULL + THEN @metadata_updates -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(@metadata_updates) AS updated + ) + )) ELSE metadata END WHERE id = @id AND state != 'running' @@ -756,7 +768,17 @@ SET THEN cast(sqlc.narg('finalized_at') AS text) ELSE finalized_at END, metadata = CASE WHEN cast(@metadata_do_merge AS boolean) AND (typeof(metadata) <> 'text' OR json_valid(metadata)) - THEN jsonb_patch(json(metadata), json(@metadata_updates)) + THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(@metadata_updates, '$.' || json_quote(member_key)) IS NOT NULL + THEN @metadata_updates -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(@metadata_updates) AS updated + ) + )) ELSE metadata END, scheduled_at = CASE WHEN /* NOT should_cancel */(cast(@state AS text) <> 'available' AND @state <> 'retryable' AND @state <> 'scheduled' OR (CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata -> 'cancel_attempted_at' END) IS NULL) AND cast(@scheduled_at_do_update AS boolean) THEN cast(@scheduled_at AS text) @@ -789,7 +811,17 @@ RETURNING -- name: JobUpdate :one UPDATE /* TEMPLATE: schema */river_job SET - metadata = CASE WHEN cast(@metadata_do_merge AS boolean) THEN jsonb_patch(json(metadata), json(@metadata)) ELSE metadata END + metadata = CASE WHEN cast(@metadata_do_merge AS boolean) THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(@metadata, '$.' || json_quote(member_key)) IS NOT NULL + THEN @metadata -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(@metadata) AS updated + ) + )) ELSE metadata END WHERE id = @id RETURNING *; diff --git a/riverdriver/riversqlite/internal/dbsqlc/river_job.sql.go b/riverdriver/riversqlite/internal/dbsqlc/river_job.sql.go index 0ff9958d2..4ca0ff39f 100644 --- a/riverdriver/riversqlite/internal/dbsqlc/river_job.sql.go +++ b/riverdriver/riversqlite/internal/dbsqlc/river_job.sql.go @@ -1801,7 +1801,17 @@ func (q *Queries) JobScheduleSetDiscarded(ctx context.Context, db DBTX, arg *Job const jobSetMetadataIfNotRunning = `-- name: JobSetMetadataIfNotRunning :one UPDATE /* TEMPLATE: schema */river_job SET metadata = CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) - THEN jsonb_patch(json(metadata), json(?1)) + THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(?1, '$.' || json_quote(member_key)) IS NOT NULL + THEN ?1 -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(?1) AS updated + ) + )) ELSE metadata END WHERE id = ?2 AND state != 'running' @@ -1856,6 +1866,8 @@ type JobSetMetadataIfNotRunningRow struct { // for JobSetStateIfRunning to use when falling back to non-running jobs. // Metadata that isn't valid JSON is left in place, and columns are listed out // to tolerate values that aren't valid JSON. See JobGetAvailable. +// Rebuild top-level members rather than using JSON Merge Patch, which removes +// JSON nulls and recursively merges nested objects. func (q *Queries) JobSetMetadataIfNotRunning(ctx context.Context, db DBTX, arg *JobSetMetadataIfNotRunningParams) (*JobSetMetadataIfNotRunningRow, error) { row := db.QueryRowContext(ctx, jobSetMetadataIfNotRunning, arg.MetadataUpdates, arg.ID) var i JobSetMetadataIfNotRunningRow @@ -1907,7 +1919,17 @@ SET THEN cast(?8 AS text) ELSE finalized_at END, metadata = CASE WHEN cast(?9 AS boolean) AND (typeof(metadata) <> 'text' OR json_valid(metadata)) - THEN jsonb_patch(json(metadata), json(?10)) + THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(?10, '$.' || json_quote(member_key)) IS NOT NULL + THEN ?10 -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(?10) AS updated + ) + )) ELSE metadata END, scheduled_at = CASE WHEN /* NOT should_cancel */(cast(?1 AS text) <> 'available' AND ?1 <> 'retryable' AND ?1 <> 'scheduled' OR (CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata -> 'cancel_attempted_at' END) IS NULL) AND cast(?11 AS boolean) THEN cast(?12 AS text) @@ -2026,7 +2048,17 @@ func (q *Queries) JobSetStateIfRunning(ctx context.Context, db DBTX, arg *JobSet const jobUpdate = `-- name: JobUpdate :one UPDATE /* TEMPLATE: schema */river_job SET - metadata = CASE WHEN cast(?1 AS boolean) THEN jsonb_patch(json(metadata), json(?2)) ELSE metadata END + metadata = CASE WHEN cast(?1 AS boolean) THEN jsonb(( + SELECT json_group_object(member_key, json(CASE + WHEN json_type(?2, '$.' || json_quote(member_key)) IS NOT NULL + THEN ?2 -> ('$.' || json_quote(member_key)) + ELSE metadata -> ('$.' || json_quote(member_key)) + END)) + FROM ( + SELECT cast(original.key AS text) AS member_key FROM json_each(CASE WHEN typeof(metadata) <> 'text' OR json_valid(metadata) THEN metadata ELSE '{}' END) AS original + UNION SELECT cast(updated.key AS text) AS member_key FROM json_each(?2) AS updated + ) + )) ELSE metadata END WHERE id = ?3 RETURNING id, json(args), attempt, attempted_at, json(attempted_by), created_at, json(errors), finalized_at, kind, max_attempts, json(metadata), priority, queue, state, scheduled_at, json(tags), unique_key, unique_states ` diff --git a/riverdriver/riversqlite/river_sqlite_driver.go b/riverdriver/riversqlite/river_sqlite_driver.go index 1fefd422a..de44e0ea3 100644 --- a/riverdriver/riversqlite/river_sqlite_driver.go +++ b/riverdriver/riversqlite/river_sqlite_driver.go @@ -73,8 +73,9 @@ func New(dbPool *sql.DB) *Driver { const argPlaceholder = "?" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameSQLite } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameSQLite } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d, nil} @@ -146,6 +147,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] )`, column, namedArg), arg, nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return "", nil, errors.New("JSON containment is not supported on SQLite") +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { arg, err := json.Marshal(values) if err != nil { diff --git a/rivermigrate/river_migrate.go b/rivermigrate/river_migrate.go index a7f9d1552..42b0acdb5 100644 --- a/rivermigrate/river_migrate.go +++ b/rivermigrate/river_migrate.go @@ -583,7 +583,7 @@ func (m *Migrator[TTx]) applyMigrations(ctx context.Context, exec riverdriver.Ex var schema string if m.schema != "" { - schema = dbutil.SafeIdentifier(m.schema) + "." + schema = m.driver.SafeIdentifier(m.schema) + "." } schemaReplacement := map[string]sqlctemplate.Replacement{ "schema": {Value: schema}, diff --git a/rivershared/riversharedtest/riversharedtest.go b/rivershared/riversharedtest/riversharedtest.go index a02143ade..662354846 100644 --- a/rivershared/riversharedtest/riversharedtest.go +++ b/rivershared/riversharedtest/riversharedtest.go @@ -112,6 +112,47 @@ var sqliteTestDir = sync.OnceValue(func() string { //nolint:gochecknoglobals return path.Join(rootDir, "sqlite") }) +// A pool and sync.Once to initialize a MySQL pool, invoked by DBPoolMySQL. +var ( + dbPoolMySQL *sql.DB //nolint:gochecknoglobals + dbPoolMySQLOnce sync.Once //nolint:gochecknoglobals + errDBPoolMySQL error //nolint:gochecknoglobals +) + +// SkipIfMySQLNotEnabled skips the current test if MySQL tests are not enabled. +// MySQL tests are opt-in because they require a running MySQL server. Set +// RIVER_MYSQL_TESTS_ENABLED=1 or RIVER_MYSQL_TESTS_ENABLED=true to enable. +func SkipIfMySQLNotEnabled(tb testing.TB) { + tb.Helper() + + val := os.Getenv("RIVER_MYSQL_TESTS_ENABLED") + if val != "1" && val != "true" { //nolint:goconst + tb.Skip("Skipping MySQL tests; set RIVER_MYSQL_TESTS_ENABLED=1 to enable") + } +} + +// DBPoolMySQL gets a lazily initialized database pool for MySQL testing. +// Uses TEST_MYSQL_URL or defaults to root@tcp(localhost:3306)/?. +func DBPoolMySQL(ctx context.Context, tb testing.TB) *sql.DB { + tb.Helper() + + dbPoolMySQLOnce.Do(func() { + dsn := cmp.Or( + os.Getenv("TEST_MYSQL_URL"), + "root@tcp(localhost:3306)/?parseTime=true&multiStatements=true&loc=UTC&time_zone=%27%2B00%3A00%27", + ) + + dbPoolMySQL, errDBPoolMySQL = sql.Open("mysql", dsn) + if errDBPoolMySQL == nil { + errDBPoolMySQL = dbPoolMySQL.PingContext(ctx) + } + }) + require.NoError(tb, errDBPoolMySQL) + require.NotNil(tb, dbPoolMySQL) + + return dbPoolMySQL +} + // DBPoolLibSQL gets a database pool appropriate for use with libSQL (a SQLite // fork) in testing. func DBPoolLibSQL(ctx context.Context, tb testing.TB, schema string) *sql.DB { diff --git a/rivershared/sqlctemplate/sqlc_template.go b/rivershared/sqlctemplate/sqlc_template.go index 1151b7d13..86fc1e115 100644 --- a/rivershared/sqlctemplate/sqlc_template.go +++ b/rivershared/sqlctemplate/sqlc_template.go @@ -89,6 +89,14 @@ type Replacement struct { // be initialized with a constructor. This lets it default to a usable instance // on drivers that may themselves not be initialized. type Replacer struct { + // UnnumberedPlaceholders, when true, causes Run to emit plain `?` + // placeholders instead of numbered `?1`, `?2`, etc. for named args + // injected via the template system. The args slice is reordered to + // match the positional order of placeholders in the SQL. This is + // needed for MySQL, whose database/sql driver does not support + // numbered `?N` syntax. + UnnumberedPlaceholders bool + cache map[replacerCacheKey]string cacheMu sync.RWMutex } @@ -142,6 +150,13 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar } cacheKey, cacheEligible := replacerCacheKeyFrom(sql, container) + + // In unnumbered mode, named args are interleaved with sqlc args during a + // left-to-right SQL walk. The cache can't reconstruct this ordering on hit. + if r.UnnumberedPlaceholders && len(container.NamedArgs) > 0 { + cacheEligible = false + } + if cacheEligible { r.cacheMu.RLock() var ( @@ -212,28 +227,36 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar } if len(container.NamedArgs) > 0 { - placeholderNum := len(args) - // For the benefit of the test suite's output being predictable, sort // named args before processing them. sortedNamedArgs := maputil.Keys(container.NamedArgs) slices.Sort(sortedNamedArgs) - for _, arg := range sortedNamedArgs { - placeholderNum++ - - var ( - symbol = "@" + arg - symbolIndex = strings.Index(updatedSQL, symbol) - val = container.NamedArgs[arg] - ) - if symbolIndex == -1 { - return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", symbol) + if r.UnnumberedPlaceholders { + var err error + updatedSQL, args, err = replaceUnnumberedArgs(updatedSQL, args, container.NamedArgs, sortedNamedArgs) + if err != nil { + return "", nil, err } + } else { + placeholderNum := len(args) + for _, arg := range sortedNamedArgs { + placeholderNum++ + + var ( + symbol = "@" + arg + symbolIndex = strings.Index(updatedSQL, symbol) + val = container.NamedArgs[arg] + ) + + if symbolIndex == -1 { + return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", symbol) + } - // ReplaceAll because an input parameter may appear multiple times. - updatedSQL = strings.ReplaceAll(updatedSQL, symbol, argPlaceholder+strconv.Itoa(placeholderNum)) - args = append(args, val) + // ReplaceAll because an input parameter may appear multiple times. + updatedSQL = strings.ReplaceAll(updatedSQL, symbol, argPlaceholder+strconv.Itoa(placeholderNum)) + args = append(args, val) + } } } @@ -249,6 +272,148 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar return updatedSQL, args, nil } +func replaceUnnumberedArgs(sql string, args []any, namedArgs map[string]any, sortedNamedArgs []string) (string, []any, error) { + // Prefer the longest name when one named argument is a prefix of another. + slices.SortFunc(sortedNamedArgs, func(left, right string) int { + switch { + case len(left) > len(right): + return -1 + case len(left) < len(right): + return 1 + default: + return strings.Compare(left, right) + } + }) + + var ( + inBlockComment bool + inLineComment bool + newArgs = make([]any, 0, len(args)+len(namedArgs)) + out strings.Builder + quote byte + sqlcArgIndex int + usedNamedArgs = make(map[string]bool, len(namedArgs)) + ) + out.Grow(len(sql)) + + for i := 0; i < len(sql); { + if inLineComment { + out.WriteByte(sql[i]) + if sql[i] == '\n' { + inLineComment = false + } + i++ + continue + } + + if inBlockComment { + if sql[i] == '*' && i+1 < len(sql) && sql[i+1] == '/' { + out.WriteString("*/") + i += 2 + inBlockComment = false + continue + } + out.WriteByte(sql[i]) + i++ + continue + } + + if quote != 0 { + out.WriteByte(sql[i]) + if sql[i] == '\\' && i+1 < len(sql) { + out.WriteByte(sql[i+1]) + i += 2 + continue + } + if sql[i] == quote { + if i+1 < len(sql) && sql[i+1] == quote { + out.WriteByte(sql[i+1]) + i += 2 + continue + } + quote = 0 + } + i++ + continue + } + + switch { + case sql[i] == '\'', sql[i] == '"', sql[i] == '`': + quote = sql[i] + out.WriteByte(sql[i]) + i++ + + case sql[i] == '#': + inLineComment = true + out.WriteByte(sql[i]) + i++ + + case sql[i] == '-' && i+2 < len(sql) && sql[i+1] == '-' && sql[i+2] <= ' ': + inLineComment = true + out.WriteString("--") + i += 2 + + case sql[i] == '/' && i+1 < len(sql) && sql[i+1] == '*': + inBlockComment = true + out.WriteString("/*") + i += 2 + + case sql[i] == '@': + matched := false + for _, name := range sortedNamedArgs { + symbol := "@" + name + if !strings.HasPrefix(sql[i:], symbol) { + continue + } + + end := i + len(symbol) + if end < len(sql) && isNamedArgChar(sql[end]) { + continue + } + + out.WriteByte('?') + newArgs = append(newArgs, namedArgs[name]) + usedNamedArgs[name] = true + i = end + matched = true + break + } + if !matched { + out.WriteByte(sql[i]) + i++ + } + + case sql[i] == '?': + if sqlcArgIndex >= len(args) { + return "", nil, errors.New("sqlctemplate found more unnumbered placeholders than sqlc arguments") + } + out.WriteByte('?') + newArgs = append(newArgs, args[sqlcArgIndex]) + sqlcArgIndex++ + i++ + + default: + out.WriteByte(sql[i]) + i++ + } + } + + if sqlcArgIndex != len(args) { + return "", nil, errors.New("sqlctemplate found fewer unnumbered placeholders than sqlc arguments") + } + for _, name := range sortedNamedArgs { + if !usedNamedArgs[name] { + return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", "@"+name) + } + } + + return out.String(), newArgs, nil +} + +func isNamedArgChar(c byte) bool { + return c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' +} + // WithReplacements adds sqlctemplate templates to the given context (they go in // context because it's the only way to get them down into the innards of sqlc). // namedArgs can also be passed in to replace arguments found in diff --git a/rivershared/sqlctemplate/sqlc_template_test.go b/rivershared/sqlctemplate/sqlc_template_test.go index 119ebd546..5441b26cd 100644 --- a/rivershared/sqlctemplate/sqlc_template_test.go +++ b/rivershared/sqlctemplate/sqlc_template_test.go @@ -259,7 +259,7 @@ func TestReplacer(t *testing.T) { AND state = @state; ` - // Initially cached value + // Initially cached value. { ctx := WithReplacements(ctx, map[string]Replacement{ "schema": {Stable: true, Value: "test_schema."}, @@ -420,6 +420,254 @@ func TestReplacer(t *testing.T) { `, updatedSQL) }) + t.Run("UnnumberedPlaceholders_NoNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Value: "test_schema."}, + }, nil) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE state = ?; + `, []any{"available"}) + require.NoError(t, err) + require.Equal(t, []any{"available"}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE state = ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgsNoInitialArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */; + `, nil) + require.NoError(t, err) + require.Equal(t, []any{"no_op"}, args) + require.Equal(t, ` + SELECT count(*) + FROM river_job + WHERE kind = ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgsWithInitialArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + // The named arg @kind appears in the WHERE clause before the + // sqlc-generated ? for LIMIT. UnnumberedPlaceholders reorders args so + // that they match the positional ? order in the final SQL. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgRepeated", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind OR queue = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + // When a named arg appears multiple times, it should produce a ? for + // each occurrence with the value duplicated in the args slice. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT * + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", "no_op", 100}, args) + require.Equal(t, ` + SELECT * + FROM river_job + WHERE kind = ? OR queue = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_MultipleNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + "where_clause": {Value: "kind = @kind AND status = @status"}, + }, map[string]any{ + "kind": "no_op", + "status": "succeeded", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", "succeeded", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? AND status = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_QuotedAndCommentedSymbols", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT '?' AS question, '@kind' AS literal + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + /* ? @kind */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT '?' AS question, '@kind' AS literal + FROM river_job + WHERE kind = ? + /* ? @kind */ + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NotCachedWithNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + "where_clause": {Stable: true, Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + sql := ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + ` + + // Unnumbered mode with named args skips caching because the + // cached SQL can't preserve the positional arg ordering. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", sql, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + + require.Empty(t, replacer.cache) + + // Second call still produces correct results. + updatedSQL, args, err = replacer.RunSafely(ctx, "?", sql, []any{200}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 200}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_CachedWithoutNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + }, nil) + + sql := ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + LIMIT ?; + ` + + // Without named args, caching works normally in unnumbered mode. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", sql, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + LIMIT ?; + `, updatedSQL) + + require.Len(t, replacer.cache, 1) + + // Second call uses cache. + updatedSQL, args, err = replacer.RunSafely(ctx, "?", sql, []any{200}) + require.NoError(t, err) + require.Equal(t, []any{200}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + LIMIT ?; + `, updatedSQL) + }) + t.Run("Stress", func(t *testing.T) { t.Parallel() diff --git a/ruby/Gemfile b/ruby/Gemfile index 494f5a94e..7ce147a9e 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -10,6 +10,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "debug" gem "fugit", "~> 1.13", require: false gem "minitest", require: false diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index 77e61c8fd..02213fb34 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -86,6 +86,8 @@ GEM drb (~> 2.0) prism (~> 1.5) mutex_m (0.3.0) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.27.0) parser (3.3.11.1) @@ -202,6 +204,7 @@ DEPENDENCIES debug fugit (~> 1.13) minitest + mysql2 (~> 0.5) pg riverqueue! riverqueue-activerecord! diff --git a/ruby/Makefile b/ruby/Makefile index f6d142fb0..afc31d6b0 100644 --- a/ruby/Makefile +++ b/ruby/Makefile @@ -7,7 +7,7 @@ TEST_JOBS ?= 4 generate/fixtures: $(MAKE) -C .. generate/fixtures -test/conformance: generate/fixtures ## Run the Go-generated fixture checks without Postgres +test/conformance: generate/fixtures ## Run Go fixture checks (SQLite by default; TEST_DATABASE=mysql for MySQL) bundle exec rspec spec/conformance_spec.rb .PHONY: test/yugabyte diff --git a/ruby/README.md b/ruby/README.md index af52500cc..fdc8e07fc 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -1,6 +1,6 @@ # River for Ruby -River is a fast, reliable background job system backed by Postgres or SQLite. The Ruby client inserts and works jobs using the same database schema and job protocol as River's other languages. It includes Active Record and Sequel drivers, plus Rails and Active Job integration. +River is a fast, reliable background job system backed by MySQL, Postgres, or SQLite. The Ruby client inserts and works jobs using the same database schema and job protocol as River's other languages. It includes Active Record and Sequel drivers, plus Rails and Active Job integration. ## Installation @@ -11,7 +11,7 @@ gem "riverqueue-sequel" gem "pg" ``` -Use `riverqueue-activerecord` for Active Record, or `sqlite3` instead of `pg` for SQLite. Install the gems with `bundle install`. Rails applications can use [`riverqueue-rails`](rails/riverqueue-rails/README.md) to configure River as their Active Job backend. +Use `riverqueue-activerecord` for Active Record, or replace `pg` with `mysql2` for MySQL or `sqlite3` for SQLite. Install the gems with `bundle install`. Rails applications can use [`riverqueue-rails`](rails/riverqueue-rails/README.md) to configure River as their Active Job backend. ## Quick start @@ -67,7 +67,7 @@ ensure end ``` -Run `bundle exec ruby worker.rb`. Replace the worker body with your mail service. Job arguments implement `kind` and `to_json`; workers read decoded JSON keys as strings. Services sharing a job must agree on its kind and JSON fields. For SQLite and alternate Postgres schemas, see [migrations](docs/migrations.md). +Run `bundle exec ruby worker.rb`. Replace the worker body with your mail service. Job arguments implement `kind` and `to_json`; workers read decoded JSON keys as strings. Services sharing a job must agree on its kind and JSON fields. For MySQL, SQLite, and alternate schemas, see [migrations](docs/migrations.md). A client without configured queues can insert jobs without starting worker threads. Applications own their database pools and should close them only after stopping River. @@ -89,3 +89,15 @@ See the [usage and configuration guide](docs/README.md), [migration guide](docs/ ## Development See [developing River for Ruby](docs/development.md). + +## MySQL + +Both Active Record and Sequel drivers support MySQL 8.0 and later. Use the latest +patch release, add `gem "mysql2"` to your bundle, and connect with a +`mysql2://user:password@localhost/my_app` URL. Active Record applications can +configure `adapter: "mysql2"` in their existing database configuration. + +The same insertion, worker, queue, and transaction APIs apply. Workers use +`FOR UPDATE SKIP LOCKED`, and River stores notification hints in the shared +MySQL outbox. Ruby's runtime polls for available work and cancellation requests. +See [migrations](docs/migrations.md) for deployment and MySQL's DDL behavior. diff --git a/ruby/docs/conformance.md b/ruby/docs/conformance.md index 9b1e8aec3..73684054b 100644 --- a/ruby/docs/conformance.md +++ b/ruby/docs/conformance.md @@ -9,9 +9,16 @@ make test/ruby/conformance ``` This generates `conformance/testdata/` and runs `ruby/spec/conformance_spec.rb`. -No Postgres server, pinned upstream checkout, protocol adapter, or separately -maintained golden files are needed. Missing fixtures fail with instructions to -regenerate them. The regular Ruby suite includes these checks too. +By default, no database server, pinned upstream checkout, protocol adapter, or +separately maintained golden files are needed. Missing fixtures fail with +instructions to regenerate them. The regular Ruby suite includes these checks too. To run the +same persistence cases on MySQL through both Active Record and Sequel: + +```sh +TEST_MYSQL_URL=mysql2://root@localhost/ make test/ruby/conformance/mysql +``` + +Each case creates and drops a disposable database using the canonical migrations. The fixture checks cover: @@ -24,13 +31,13 @@ The fixture checks cover: - Cron occurrences, named zones, daylight-saving transitions, interval schedules, and invalid expressions, with the Ruby API differences below checked explicitly. - Insertion, cancellation, queue pause/resume/metadata, and leadership resignation - notification encoding through both SQL drivers, using in-memory SQLite and + notification encoding through both SQL drivers, using SQLite or MySQL and the bundled canonical migrations. The Ruby workflow runs on Ruby changes. The shared Conformance workflow also -runs these tests when Go code changes. The ordinary driver suites retain -Postgres notification delivery/commit-ordering tests and shared Postgres / -SQLite insertion, transaction, uniqueness, and worker tests. +runs these tests on SQLite and MySQL when Go code changes. The ordinary driver +suites retain Postgres notification delivery/commit-ordering tests and shared +Postgres / SQLite / MySQL insertion, transaction, uniqueness, and worker tests. This replaces the old `insert-only-v1` adapter and its pinned external harness. It checks shared data formats, not live mixed-language worker execution. Adapter @@ -40,7 +47,7 @@ protocol is not part of the Ruby API. ## Coverage gaps and API differences Ruby's runtime still polls job and queue state rather than consuming Postgres -LISTEN events or the SQLite outbox. The fixture target checks notification +LISTEN events or the SQLite/MySQL outbox. The fixture target checks notification emission, not dispatch. Ruby does not emit or handle `request_resign`; other clients' requests therefore cannot force a Ruby leader to resign. Completing that coverage would require a notification listener and runtime dispatch path. @@ -49,7 +56,7 @@ rollback for queue controls and resignations as well as insertion/cancellation. Shared snooze-counter fixtures cover non-negative integers and absent counters. Recovery from other JSON values is implementation-specific. Ruby's shared driver -tests cover its lenient conversion rules separately on Postgres and SQLite. +tests cover its lenient conversion rules separately on Postgres, SQLite, and MySQL. Cron retains Ruby's existing Fugit extensions: six fields with seconds, Sunday as 7, hour 24, and descending ranges. These four expressions are accepted in diff --git a/ruby/docs/development.md b/ruby/docs/development.md index 8fd744420..24c19f5d2 100644 --- a/ruby/docs/development.md +++ b/ruby/docs/development.md @@ -174,3 +174,9 @@ repository. Follow its README; do not include it in the public release below. ``` 4. Cut a new GitHub release by visiting [new release](https://github.com/riverqueue/river/releases/new), selecting the new tag, and copying in the version's `CHANGELOG.md` content as the release body. + +For MySQL, set `TEST_DATABASE=mysql` and +`TEST_MYSQL_URL=mysql2://root@127.0.0.1/`, then run `make spec/core` from `ruby/`. +The test account must be able to create and drop disposable databases. CI runs +the shared driver, worker, and migration contracts on MySQL 8.0 and 8.4 and +includes their coverage in the combined report. diff --git a/ruby/docs/migrations.md b/ruby/docs/migrations.md index cd7e3bc21..0db907173 100644 --- a/ruby/docs/migrations.md +++ b/ruby/docs/migrations.md @@ -1,7 +1,7 @@ # Schema migrations River Ruby includes a migration API and the `river` command. The gems bundle -byte-for-byte copies of River's canonical Postgres and SQLite migration SQL, +byte-for-byte copies of River's canonical MySQL, Postgres, and SQLite migration SQL, pinned to release v0.48.0 through migration 008; only the upstream schema template placeholders are substituted when executing them. @@ -9,7 +9,7 @@ them. ## Command line Install `riverqueue-sequel` or `riverqueue-activerecord` and the application's -database gem (`pg` or `sqlite3`). The command auto-detects the River driver gem +database gem (`mysql2`, `pg`, or `sqlite3`). The command auto-detects the River driver gem available in your bundle, preferring Sequel when both are available. This choice does not need to match your application's driver: both run the same migrations. Create the database first, then run: @@ -18,6 +18,9 @@ Create the database first, then run: bundle exec river migrate-status --database-url postgres://localhost/my_app bundle exec river migrate-up --database-url postgres://localhost/my_app +# MySQL, using either driver: +bundle exec river migrate-up --database-url mysql2://localhost/my_app + # SQLite, using Sequel: bundle exec river migrate-up --database-url sqlite://storage/river.sqlite3 ``` @@ -52,14 +55,18 @@ migrator.migrate(dry_run: true, target: 8) migrator.migrate(direction: :down) # One version; no interactive confirmation. ``` -The API accepts either driver and optionally `schema:` for Postgres. `migrate` +The API accepts either driver and optionally `schema:` for MySQL or Postgres. `migrate` returns the migration records applied (or planned in dry-run mode), with their -version, name, and original up/down SQL. Each version's SQL and history update +version, name, and original up/down SQL. On Postgres and SQLite, each version's SQL and history update commit in their own transaction. A failing version rolls back while earlier versions remain committed, so rerunning resumes safely. Do not wrap the migrator in an application transaction: some Postgres changes require a real commit before the next version can run. +MySQL DDL commits implicitly. History is recorded after each successful version, +but a failure can leave partially applied DDL. Inspect and repair that version +before retrying. MySQL migrators hold a named lock for the selected database. + An up migration targeting an already-applied version does nothing, even when newer migrations are pending. diff --git a/ruby/driver/riverqueue-activerecord/Gemfile b/ruby/driver/riverqueue-activerecord/Gemfile index fbff1c6b2..ed14420dc 100644 --- a/ruby/driver/riverqueue-activerecord/Gemfile +++ b/ruby/driver/riverqueue-activerecord/Gemfile @@ -10,6 +10,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "debug" gem "pg" gem "rspec-core" diff --git a/ruby/driver/riverqueue-activerecord/Gemfile.lock b/ruby/driver/riverqueue-activerecord/Gemfile.lock index 000139b90..3e08f6cea 100644 --- a/ruby/driver/riverqueue-activerecord/Gemfile.lock +++ b/ruby/driver/riverqueue-activerecord/Gemfile.lock @@ -65,6 +65,8 @@ GEM minitest (6.0.2) drb (~> 2.0) prism (~> 1.5) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.27.0) parser (3.3.11.1) @@ -153,6 +155,7 @@ PLATFORMS DEPENDENCIES debug + mysql2 (~> 0.5) pg riverqueue! riverqueue-activerecord! diff --git a/ruby/driver/riverqueue-activerecord/lib/driver.rb b/ruby/driver/riverqueue-activerecord/lib/driver.rb index fab5ab795..7fc94bf80 100644 --- a/ruby/driver/riverqueue-activerecord/lib/driver.rb +++ b/ruby/driver/riverqueue-activerecord/lib/driver.rb @@ -4,7 +4,7 @@ module River::Driver # Provides an ActiveRecord driver for River supporting Postgres, YugabyteDB, - # and SQLite. + # MySQL, and SQLite. # # Used in conjunction with a River client like: # @@ -17,7 +17,7 @@ class ActiveRecord # Connection class whose pool and transaction context this driver uses. attr_reader :connection_class - # Uses an established Postgres or SQLite connection. The class must be + # Uses an established Postgres, MySQL, or SQLite connection. The class must be # ActiveRecord::Base or an abstract Active Record class. Routing follows its # current role/shard; configure consumers explicitly for each database. def initialize(connection_class: ::ActiveRecord::Base) @@ -32,6 +32,8 @@ def initialize(connection_class: ::ActiveRecord::Base) @connection_class = connection_class @postgres_capabilities_cache = PostgresCapabilities::Cache.new @is_sqlite = connection_class.connection.adapter_name.downcase.include?("sqlite") + @is_mysql = connection_class.connection.adapter_name.downcase.include?("mysql") + extend River::Driver::MySQLRuntime if @is_mysql # Do not inherit application scopes, callbacks, or schema caches. Each # driver owns its model, but all operations use the selected class's pool. @job_model = Class.new(::ActiveRecord::Base) do @@ -85,7 +87,11 @@ def rollback_exception end # Backend used by River::Migrator. - def migration_backend = @is_sqlite ? :sqlite : :postgresql + def migration_backend = if @is_mysql + :mysql + else + (@is_sqlite ? :sqlite : :postgresql) + end # Pins a raw connection for migration SQL and refreshes cached model columns. def migration_connection @@ -240,7 +246,7 @@ def transaction(&) end private def runtime_postgres? - !@is_sqlite + !@is_sqlite && !@is_mysql end private def runtime_query_rows(sql) diff --git a/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb b/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb index 124d96abf..3a09add1f 100644 --- a/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb +++ b/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb @@ -5,6 +5,16 @@ require_relative "../../../spec/row_decoding_shared_examples" RSpec.describe River::Driver::ActiveRecord do + it "selects the MySQL runtime and migrations for mysql2 connections" do + connection_class = Class.new(ActiveRecord::Base) do + self.abstract_class = true + def self.connection = Struct.new(:adapter_name).new("Mysql2") + end + driver = described_class.new(connection_class: connection_class) + expect(driver).to be_a(River::Driver::MySQLRuntime) + expect(driver.migration_backend).to eq(:mysql) + end + before do if ENV["RIVER_DEBUG"] == "1" || ENV["RIVER_DEBUG"] == "true" ActiveRecord::Base.logger = Logger.new($stdout) diff --git a/ruby/driver/riverqueue-sequel/Gemfile b/ruby/driver/riverqueue-sequel/Gemfile index 606177f7c..f9777d6c6 100644 --- a/ruby/driver/riverqueue-sequel/Gemfile +++ b/ruby/driver/riverqueue-sequel/Gemfile @@ -10,6 +10,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "pg" gem "rspec-core" gem "rspec-expectations" diff --git a/ruby/driver/riverqueue-sequel/Gemfile.lock b/ruby/driver/riverqueue-sequel/Gemfile.lock index 95ff36b51..56bb6c8fc 100644 --- a/ruby/driver/riverqueue-sequel/Gemfile.lock +++ b/ruby/driver/riverqueue-sequel/Gemfile.lock @@ -25,6 +25,8 @@ GEM language_server-protocol (3.17.0.5) lint_roller (1.1.0) logger (1.7.0) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.27.0) parser (3.3.11.1) @@ -97,6 +99,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + mysql2 (~> 0.5) pg riverqueue! riverqueue-sequel! diff --git a/ruby/driver/riverqueue-sequel/lib/driver.rb b/ruby/driver/riverqueue-sequel/lib/driver.rb index 354c33998..027360244 100644 --- a/ruby/driver/riverqueue-sequel/lib/driver.rb +++ b/ruby/driver/riverqueue-sequel/lib/driver.rb @@ -3,7 +3,7 @@ require "securerandom" module River::Driver - # Provides a Sequel driver for River supporting Postgres, YugabyteDB, and SQLite. + # Provides a Sequel driver for River supporting Postgres, YugabyteDB, MySQL, and SQLite. # # Used in conjunction with a River client like: # @@ -18,14 +18,16 @@ module River::Driver class Sequel include River::Driver::Runtime - # Creates a driver backed by a connected Sequel::Database for Postgres or + # Creates a driver backed by a connected Sequel::Database for Postgres, MySQL, or # SQLite. def initialize(db) @db = db @postgres_capabilities_cache = PostgresCapabilities::Cache.new @is_sqlite = (db.database_type == :sqlite) + @is_mysql = (db.database_type == :mysql) + extend River::Driver::MySQLRuntime if @is_mysql - unless @is_sqlite + unless @is_sqlite || @is_mysql db.extension(:pg_array) db.extension(:pg_json) end @@ -68,7 +70,11 @@ def rollback_exception end # Backend used by River::Migrator. - def migration_backend = @is_sqlite ? :sqlite : :postgresql + def migration_backend = if @is_mysql + :mysql + else + (@is_sqlite ? :sqlite : :postgresql) + end # Pins a raw connection for the complete migration operation. def migration_connection @@ -210,7 +216,7 @@ def transaction end private def runtime_postgres? - !@is_sqlite + !@is_sqlite && !@is_mysql end private def runtime_query_rows(sql) diff --git a/ruby/driver/riverqueue-sequel/spec/driver_spec.rb b/ruby/driver/riverqueue-sequel/spec/driver_spec.rb index ce1b4ce82..205bd433f 100644 --- a/ruby/driver/riverqueue-sequel/spec/driver_spec.rb +++ b/ruby/driver/riverqueue-sequel/spec/driver_spec.rb @@ -5,6 +5,14 @@ require_relative "../../../spec/row_decoding_shared_examples" RSpec.describe River::Driver::Sequel do + it "selects the MySQL runtime and migrations for mysql2 connections" do + database = Sequel.mock + def database.database_type = :mysql + driver = described_class.new(database) + expect(driver).to be_a(River::Driver::MySQLRuntime) + expect(driver.migration_backend).to eq(:mysql) + end + if DB context "with Postgres", database: :postgres do around(:each) { |ex| test_transaction(&ex) } diff --git a/ruby/lib/driver.rb b/ruby/lib/driver.rb index 3be96401c..d0ca3ef07 100644 --- a/ruby/lib/driver.rb +++ b/ruby/lib/driver.rb @@ -51,3 +51,4 @@ def initialize( require_relative "driver/postgres_capabilities" require_relative "driver/job_row_decoder" require_relative "driver/runtime" +require_relative "driver/mysql_runtime" diff --git a/ruby/lib/driver/mysql_runtime.rb b/ruby/lib/driver/mysql_runtime.rb new file mode 100644 index 000000000..a8df9529d --- /dev/null +++ b/ruby/lib/driver/mysql_runtime.rb @@ -0,0 +1,343 @@ +# frozen_string_literal: true + +module River::Driver + # MySQL statements shared by the Active Record and Sequel adapters. Writes + # lock and read back their rows in the same transaction because MySQL has no + # RETURNING clause. Only MySQL driver instances extend this module. + module MySQLRuntime + def job_cancel(id, now: Time.now.utc) + transaction do + rows = mysql_update("river_job", "id", "id = #{Integer(id)} AND state NOT IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NULL", [ + "finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE #{runtime_time(now)} END", + "metadata = #{runtime_merge_metadata("cancel_attempted_at" => now.iso8601(6))}", + "state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END" + ]) + job = job_get_by_id(id) + runtime_notify("river_control", action: "cancel", job_id: job.id, queue: job.queue) unless rows.empty? + job + end + end + + def job_complete(id:, finalized_at:, metadata: nil, now: Time.now.utc) + assignments = ["finalized_at = #{runtime_time(finalized_at)}", "state = 'completed'"] + assignments << "metadata = #{runtime_merge_metadata(metadata)}" if metadata && !metadata.empty? + transaction do + rows = mysql_update("river_job", "id", "id = #{Integer(id)} AND state = 'running' AND NOT (#{runtime_cancel_attempted})", assignments) + next runtime_decode_sqlite_job(rows.first) unless rows.empty? + + :cancelled if job_get_cancelled_ids([Integer(id)]).include?(Integer(id)) + end + end + + def job_delete(id) + transaction do + rows = mysql_delete("river_job", "id", "id = #{Integer(id)} AND state != 'running'") + rows.empty? ? job_get_by_id(id) : runtime_decode_sqlite_job(rows.first) + end + end + + def job_delete_finalized(retention:, now: Time.now.utc, max: 1_000) + clauses = retention.filter_map do |state, seconds| + "(state = #{runtime_quote(state.to_s)} AND finalized_at < #{runtime_time(now - seconds)})" if seconds + end + return 0 if clauses.empty? + + transaction do + predicate = "(#{clauses.join(" OR ")})" + ids = runtime_query_rows("SELECT id FROM river_job WHERE #{predicate} ORDER BY id LIMIT #{Integer(max)} FOR UPDATE SKIP LOCKED") + .map { |row| runtime_value(row, :id) } + next 0 if ids.empty? + + mysql_delete("river_job", "id", "#{runtime_in_clause("id", ids)} AND #{predicate}").length + end + end + + def job_delete_if_running(id) + transaction do + deleted = mysql_delete("river_job", "id", "id = #{Integer(id)} AND state = 'running' AND NOT (#{runtime_cancel_attempted})") + next true unless deleted.empty? + + job_get_cancelled_ids([Integer(id)]).include?(Integer(id)) ? :cancelled : false + end + end + + def job_delete_many(params) + transaction do + jobs = runtime_job_list(params, for_delete: true) + next [] if jobs.empty? + + mysql_delete("river_job", "id", runtime_in_clause("id", jobs.map(&:id)) + " AND state != 'running'") + jobs + end + end + + def job_get_by_id(id) + row = runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job WHERE id = #{Integer(id)}").first + row ? runtime_decode_sqlite_job(row) : nil + end + + def job_insert_many(params) + return [] if params.empty? + + nonces = sqlite_insert_nonces(params) + transaction do + results = params.zip(nonces).map do |param, nonce| + metadata = (param.metadata || {}).merge("river:unique_nonce" => nonce) + values = [runtime_json(param.encoded_args), runtime_quote(param.kind), Integer(param.max_attempts), + runtime_json(metadata), Integer(param.priority), runtime_quote(param.queue), + param.scheduled_at ? runtime_time(param.scheduled_at) : "UTC_TIMESTAMP(6)", + runtime_quote(param.state), runtime_json(param.tags || []), + param.unique_key ? "UNHEX(#{runtime_quote(param.unique_key.unpack1("H*"))})" : "NULL", + param.unique_states ? param.unique_states.to_i(2) : "NULL"] + runtime_execute(<<~SQL) + INSERT INTO river_job (args, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) + VALUES (#{values.join(",")}) + ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) + SQL + id = runtime_value(runtime_query_rows("SELECT LAST_INSERT_ID() AS id").first, :id) + job = job_get_by_id(id) + duplicate = job.metadata["river:unique_nonce"] != nonce + if duplicate && (param.unique_key.nil? || job.unique_key != param.unique_key || + !job.unique_states&.include?(job.state) || + !River::UniqueBitmask.to_states((param.unique_states || "0").to_i(2)).include?(param.state)) + raise River::Error, "insert conflicts with a constraint other than River's unique-job index" + end + [job, duplicate] + end + params.select { |param| param.state == River::JOB_STATE_AVAILABLE }.map(&:queue).uniq.each do |queue| + runtime_notify("river_insert", queue: queue) + end + results + end + end + + def job_metadata_merge(id, metadata) + rows = mysql_update("river_job", "id", "id = #{Integer(id)}", ["metadata = #{runtime_merge_metadata(metadata)}"]) + rows.empty? ? nil : runtime_decode_sqlite_job(rows.first) + end + + def job_retry(id, now: Time.now.utc) + transaction do + job = mysql_locked_job(id) + next job if !job || job.state == "running" || (job.state == "available" && job.scheduled_at <= now) + raise ArgumentError, "cannot retry a job with #{River::MAX_ATTEMPTS_LIMIT} or more attempts" if job.attempt >= River::MAX_ATTEMPTS_LIMIT + + mysql_update("river_job", "id", "id = #{Integer(id)}", [ + "max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END", + "metadata = JSON_REMOVE(metadata, '$.cancel_attempted_at')", "finalized_at = NULL", + "scheduled_at = #{runtime_time(now)}", "state = 'available'" + ]) + job_get_by_id(id) + end + end + + def job_set_state_if_running(id:, state:, now: Time.now.utc, attempt: nil, + error: nil, finalized_at: nil, metadata: nil, scheduled_at: nil) + cancel = runtime_cancel_attempted + assignments = [] #: Array[String] + assignments << "attempt = CASE WHEN NOT (#{cancel}) THEN #{Integer(attempt)} ELSE attempt END" unless attempt.nil? + assignments << "errors = #{runtime_append_error(error)}" if error + assignments << "finalized_at = CASE WHEN #{cancel} THEN #{runtime_time(now)} ELSE #{runtime_nullable_time(finalized_at)} END" + assignments << "scheduled_at = CASE WHEN NOT (#{cancel}) THEN #{runtime_time(scheduled_at)} ELSE scheduled_at END" if scheduled_at + # MySQL evaluates assignments left-to-right. Check cancellation before + # replacing metadata or state. + assignments << "state = CASE WHEN #{cancel} THEN 'cancelled' ELSE #{runtime_state(state.to_s)} END" + assignments << "metadata = #{runtime_merge_metadata(metadata)}" if metadata && !metadata.empty? + rows = mysql_update("river_job", "id", "id = #{Integer(id)} AND state = 'running'", assignments) + rows.empty? ? nil : runtime_decode_sqlite_job(rows.first) + end + + def job_update(id, params) + assignments = params.each.map do |field, value| + raise ArgumentError, "unknown update field: #{field}" unless River::JobUpdateParams.method_defined?(field) + + "#{field} = #{runtime_update_value(field, value)}" + end + return job_get_by_id(id) if assignments.empty? + + rows = mysql_update("river_job", "id", "id = #{Integer(id)}", assignments) + rows.empty? ? nil : runtime_decode_sqlite_job(rows.first) + end + + def leader_acquire(id, ttl: 30, now: Time.now.utc) + transaction do + runtime_execute("DELETE FROM river_leader WHERE expires_at < #{runtime_time(now)}") + runtime_execute(<<~SQL) + INSERT INTO river_leader (leader_id, elected_at, expires_at) + VALUES (#{runtime_quote(id)}, #{runtime_time(now)}, #{runtime_time(now + ttl)}) + ON DUPLICATE KEY UPDATE name = name + SQL + runtime_query_rows("SELECT leader_id FROM river_leader WHERE leader_id = CAST(#{runtime_quote(id)} AS BINARY) FOR UPDATE").any? + end + end + + def leader_release(id) + transaction do + deleted = mysql_delete("river_leader", "name", "leader_id = CAST(#{runtime_quote(id)} AS BINARY)") + runtime_notify("river_leadership", action: "resigned", leader_id: id) unless deleted.empty? + end + end + + def leader_renew(id, ttl: 30, now: Time.now.utc) + mysql_update("river_leader", "name", "leader_id = CAST(#{runtime_quote(id)} AS BINARY) AND expires_at >= #{runtime_time(now)}", + ["expires_at = #{runtime_time(now + ttl)}"]).any? + end + + def notification_delete_before(horizon:, max: 10_000) + mysql_delete("river_notification", "id", "created_at < #{runtime_time(horizon)}", limit: max).length + end + + def queue_pause(name, now: Time.now.utc) + mysql_queue_paused(name, true, now) + end + + def queue_resume(name, now: Time.now.utc) + mysql_queue_paused(name, false, now) + end + + def queue_update(name, metadata:, now: Time.now.utc) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + transaction do + rows = mysql_update("river_queue", "name", "name = CAST(#{runtime_quote(name)} AS BINARY)", + ["metadata = #{runtime_json(metadata)}", "updated_at = #{runtime_time(now)}"]) + next nil if rows.empty? + + runtime_notify("river_control", action: "metadata_changed", queue: name, metadata: metadata) + runtime_queue_from_row(rows.first) + end + end + + def queue_upsert(name, metadata: {}, now: Time.now.utc) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + transaction do + runtime_execute(<<~SQL) + INSERT INTO river_queue (name, created_at, metadata, updated_at) + VALUES (#{runtime_quote(name)}, #{runtime_time(now)}, #{runtime_json(metadata)}, #{runtime_time(now)}) + ON DUPLICATE KEY UPDATE updated_at = #{runtime_time(now)} + SQL + queue_get(name) + end + end + + private def mysql_columns(table) + columns = case table + when "river_job" + %w[id args attempt attempted_at attempted_by created_at errors finalized_at kind max_attempts metadata priority queue scheduled_at state tags unique_key unique_states] + when "river_queue" + %w[name created_at metadata paused_at updated_at] + else + return "*" + end + columns.map { |column| column.end_with?("_at") ? "CAST(#{column} AS CHAR) AS #{column}" : column }.join(", ") + end + + private def mysql_delete(table, key, predicate, limit: nil) + transaction do + hint = (table == "river_job") ? "/*+ JOIN_INDEX(river_job PRIMARY) */ " : "" + rows = runtime_query_rows("SELECT #{hint}#{mysql_columns(table)} FROM #{table} WHERE #{predicate} ORDER BY #{key} #{"LIMIT #{Integer(limit)}" if limit} FOR UPDATE") + runtime_execute("DELETE /*+ JOIN_INDEX(#{table} PRIMARY) */ FROM #{table} WHERE #{runtime_in_clause(key, rows.map { |row| runtime_value(row, key.to_sym) })}") unless rows.empty? + rows + end + end + + private def mysql_locked_job(id) + row = runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job WHERE id = #{Integer(id)} FOR UPDATE").first + row ? runtime_decode_sqlite_job(row) : nil + end + + private def mysql_queue_paused(name, paused, now) + transaction do + predicate = (name == "*") ? "true" : "name = CAST(#{runtime_quote(name)} AS BINARY)" + assignments = ["updated_at = CASE WHEN paused_at IS #{paused ? "NULL" : "NOT NULL"} THEN #{runtime_time(now)} ELSE updated_at END", + "paused_at = #{paused ? "COALESCE(paused_at, #{runtime_time(now)})" : "NULL"}"] + rows = mysql_update("river_queue", "name", predicate, assignments) + runtime_notify("river_control", action: paused ? "pause" : "resume", queue: name) unless rows.empty? + rows.map { |row| runtime_queue_from_row(row) } + end + end + + private def mysql_update(table, key, predicate, assignments) + transaction do + rows = runtime_query_rows("SELECT #{key} FROM #{table} WHERE #{predicate} ORDER BY #{key} FOR UPDATE") + next [] if rows.empty? + + filter = runtime_in_clause(key, rows.map { |row| runtime_value(row, key.to_sym) }) + runtime_execute("UPDATE #{table} SET #{assignments.join(", ")} WHERE #{filter}") + runtime_query_rows("SELECT #{mysql_columns(table)} FROM #{table} WHERE #{filter} ORDER BY #{key}") + end + end + + private def runtime_append_error(error) + value = error.respond_to?(:to_h) ? error.to_h : error + "JSON_ARRAY_APPEND(CASE WHEN JSON_TYPE(errors) = 'ARRAY' THEN errors WHEN errors IS NULL THEN JSON_ARRAY() ELSE JSON_ARRAY(errors) END, '$', #{runtime_json(value)})" + end + + private def runtime_cancel_attempted = "JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')" + + private def runtime_claim_jobs(predicate, max, attempted_by, now) + transaction do + ids = runtime_query_rows("SELECT id FROM river_job WHERE #{predicate} ORDER BY priority, scheduled_at, id LIMIT #{Integer(max)} FOR UPDATE SKIP LOCKED") + .map { |row| runtime_value(row, :id) } + next [] if ids.empty? + + runtime_execute(<<~SQL) + UPDATE river_job SET attempt = LEAST(attempt + 1, 32767), + attempted_at = #{runtime_time(now)}, + attempted_by = JSON_ARRAY_APPEND(CASE WHEN JSON_LENGTH(attempted_by) >= 100 THEN JSON_EXTRACT(attempted_by, '$[last-98 to last]') ELSE COALESCE(attempted_by, JSON_ARRAY()) END, '$', #{runtime_quote(attempted_by)}), + state = 'running' + WHERE #{runtime_in_clause("id", ids)} + SQL + ids.map { |id| runtime_read_job(id) } + end + end + + private def runtime_in_clause(column, values) + predicate = super + return predicate unless %w[kind name queue].include?(column) + + # Keep the indexed comparison so claims do not scan and lock other queues. + "(#{predicate} AND #{super("CAST(#{column} AS BINARY)", values)})" + end + + private def runtime_job_rows(suffix) + runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job #{suffix}").map { |row| runtime_decode_sqlite_job(row) } + end + + private def runtime_json(value) + encoded = value.is_a?(String) ? value : JSON.generate(value) + "CAST(#{runtime_quote(encoded)} AS JSON)" + end + + private def runtime_locks_rows? = true + + private def runtime_merge_metadata(metadata) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + metadata.reduce("metadata") do |expression, (key, value)| + "JSON_SET(#{expression}, #{runtime_quote("$.#{JSON.generate(key.to_s)}")}, CAST(#{runtime_quote(JSON.generate(value))} AS JSON))" + end + end + + private def runtime_metadata_equals(key, value) + "JSON_EXTRACT(metadata, #{runtime_quote("$.#{JSON.generate(key.to_s)}")}) = CAST(#{runtime_quote(JSON.generate(value))} AS JSON)" + end + + private def runtime_order_by(params, null_order) + "#{"#{params.sort_by} IS NULL, " unless null_order.empty?}#{params.sort_by} #{params.sort_order.to_s.upcase}, id #{params.sort_order.to_s.upcase}" + end + + private def runtime_queue_columns = mysql_columns("river_queue") + + private def runtime_tag_contains(tag) + "JSON_CONTAINS(tags, JSON_QUOTE(#{runtime_quote(tag)}))" + end + + private def runtime_time(value) + raise ArgumentError, "time cannot be nil" unless value + + runtime_quote(value.getutc.strftime("%Y-%m-%d %H:%M:%S.%6N")) + end + end +end diff --git a/ruby/lib/driver/runtime.rb b/ruby/lib/driver/runtime.rb index 93d942981..ef000e6c6 100644 --- a/ruby/lib/driver/runtime.rb +++ b/ruby/lib/driver/runtime.rb @@ -146,7 +146,7 @@ def job_delete_many(params) def job_get_available(queue:, max:, attempted_by:, kinds: nil, now: Time.now.utc) return [] if kinds&.empty? - predicate = "state = 'available' AND queue = #{runtime_quote(queue)} AND scheduled_at <= #{runtime_time(now)}" + predicate = "state = 'available' AND #{runtime_in_clause("queue", [queue])} AND scheduled_at <= #{runtime_time(now)}" predicate += " AND #{runtime_in_clause("kind", kinds)}" if kinds runtime_claim_jobs(predicate, max, attempted_by, now) end @@ -181,7 +181,7 @@ def job_rescue_stuck(horizon:, retry_policy:, now: Time.now.utc, max: 1_000, log # Lock each candidate until its transition finishes. Continue past jobs # that are still within their timeouts without consuming the rescue limit. after_id = rescued = 0 - lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : "" + lock = runtime_locks_rows? ? "FOR UPDATE SKIP LOCKED" : "" while rescued < max ids = runtime_returning_ids(<<~SQL) SELECT id FROM river_job @@ -256,7 +256,7 @@ def job_schedule(now: Time.now.utc, max: 1_000) transaction do # Hold each selected row until its transition (including uniqueness # conflict handling) finishes. A concurrent retry may change its due time. - lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : "" + lock = runtime_locks_rows? ? "FOR UPDATE SKIP LOCKED" : "" ids = runtime_query_rows(<<~SQL).map { |row| runtime_value(row, :id).to_i } SELECT id FROM river_job WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= #{runtime_time(now)} @@ -366,7 +366,7 @@ def notification_delete_before(horizon:, max: 10_000) end def queue_get(name) - row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE name = #{runtime_quote(name)}").first + row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE #{runtime_in_clause("name", [name])}").first runtime_queue_from_row(row) end @@ -473,7 +473,7 @@ def queue_upsert(name, metadata: {}, now: Time.now.utc) SQL end - lock_clause = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : "" + lock_clause = runtime_locks_rows? ? "FOR UPDATE SKIP LOCKED" : "" # An administratively requeued job may already be at the counter limit. # Saturate instead of letting that row abort the entire claim batch. ids = runtime_returning_ids(<<~SQL) @@ -609,12 +609,18 @@ def queue_upsert(name, metadata: {}, now: Time.now.utc) where = clauses.empty? ? "" : "WHERE #{clauses.join(" AND ")}" runtime_job_rows(<<~SQL) #{where} - ORDER BY #{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase} + ORDER BY #{runtime_order_by(params, null_order)} LIMIT #{params.limit} - #{"FOR UPDATE SKIP LOCKED" if for_delete && runtime_postgres?} + #{"FOR UPDATE SKIP LOCKED" if for_delete && runtime_locks_rows?} SQL end + private def runtime_locks_rows? = runtime_postgres? + + private def runtime_order_by(params, null_order) + "#{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase}" + end + private def runtime_json(value) encoded = value.is_a?(String) ? value : JSON.generate(value) runtime_postgres? ? "#{runtime_quote(encoded)}::jsonb" : "jsonb(#{runtime_quote(encoded)})" diff --git a/ruby/lib/migrator.rb b/ruby/lib/migrator.rb index a17ddfcdc..22bd5f083 100644 --- a/ruby/lib/migrator.rb +++ b/ruby/lib/migrator.rb @@ -15,7 +15,7 @@ class Migrator def initialize(driver, line: "main", migrations_path: File.expand_path("../migration", __dir__.to_s), schema: nil) @backend = driver.migration_backend @driver = driver - raise ArgumentError, "unsupported migration backend" unless [:postgresql, :sqlite].include?(@backend) + raise ArgumentError, "unsupported migration backend" unless [:mysql, :postgresql, :sqlite].include?(@backend) raise ArgumentError, "invalid migration line" unless line.match?(/\A[a-z][a-z0-9_]*\z/) raise ArgumentError, "SQLite does not support a migration schema option" if @backend == :sqlite && schema @@ -61,7 +61,7 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) unless dry_run plan.each do |migration| - execute((@backend == :sqlite) ? "BEGIN IMMEDIATE" : "BEGIN") + execute((@backend == :sqlite) ? "BEGIN IMMEDIATE" : "BEGIN") unless @backend == :mysql committed = false begin raise River::Error, "migration state changed concurrently; rerun migration" unless existing_versions == existing @@ -71,7 +71,7 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) end sql = (direction == :up) ? migration.sql_up : migration.sql_down - execute(sql.gsub("/* TEMPLATE: schema */", @schema ? %("#{@schema}".) : "")) + execute(sql.gsub("/* TEMPLATE: schema */", schema_prefix)) if direction == :up columns, values = line_column? ? ["line, version", "'#{@line}', #{migration.version}"] : ["version", migration.version.to_s] @@ -86,10 +86,10 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) existing -= [migration.version] end - execute("COMMIT") + execute("COMMIT") unless @backend == :mysql committed = true ensure - execute("ROLLBACK") unless committed + execute("ROLLBACK") unless committed || @backend == :mysql end end end @@ -109,6 +109,10 @@ def status private def execute(sql) if @backend == :postgresql @connection.exec(sql) + elsif @backend == :mysql + # MySQL DDL commits implicitly. Run statements separately without + # requiring MULTI_STATEMENTS on the application's connection pool. + mysql_statements(sql).each { |statement| @connection.query(statement) } else @connection.execute_batch(sql) end @@ -117,6 +121,8 @@ def status private def existing_versions exists = if @backend == :postgresql query("SELECT to_regclass('#{table}') AS name").first.fetch("name") + elsif @backend == :mysql + query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{@schema}' AND table_name = 'river_migration'").any? else query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'river_migration'").any? end @@ -138,7 +144,7 @@ def status end private def line_column? - if @backend == :postgresql + if [:mysql, :postgresql].include?(@backend) query("SELECT column_name FROM information_schema.columns WHERE table_schema = '#{@schema}' AND table_name = 'river_migration' AND column_name = 'line'").any? else query("PRAGMA table_info(river_migration)").any? { |row| row.fetch("name") == "line" } @@ -148,6 +154,8 @@ def status private def query(sql) if @backend == :postgresql @connection.exec(sql).to_a + elsif @backend == :mysql + @connection.query(sql, as: :hash, symbolize_keys: false).to_a else rows = @connection.execute2(sql) columns = rows.shift @@ -161,6 +169,16 @@ def status @connection = connection @schema = if @backend == :postgresql @requested_schema || query("SELECT current_schema() AS name").first.fetch("name") + elsif @backend == :mysql + @requested_schema || query("SELECT DATABASE() AS name").first.fetch("name") + end + if @backend == :mysql + raise ArgumentError, "schema must be a simple SQL identifier" unless @schema&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]{0,45}\z/) + + if lock + locked = query("SELECT GET_LOCK('river_migrate:#{@schema}', 0) AS locked").first.fetch("locked") + raise River::Error, "another migrator holds the database lock" unless locked == 1 + end end if @backend == :postgresql raise ArgumentError, "schema must be a simple SQL identifier" unless @schema&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/) @@ -176,6 +194,8 @@ def status ensure if @backend == :postgresql && lock query("SELECT pg_advisory_unlock(hashtext(current_database()), hashtext('river_migrate:#{@schema}'))") + elsif @backend == :mysql && lock + query("SELECT RELEASE_LOCK('river_migrate:#{@schema}')") end end end @@ -183,7 +203,33 @@ def status end private def table - @schema ? %("#{@schema}".river_migration) : "river_migration" + "#{schema_prefix}river_migration" + end + + private def schema_prefix + return "" unless @schema + + (@backend == :mysql) ? "`#{@schema}`." : %("#{@schema}".) + end + + private def mysql_statements(sql) + statements = [] #: Array[String] + buffer = +"" + # Quoted text and comments are tokens so their semicolons cannot split + # statements. River migrations contain plain SQL, not DELIMITER commands. + sql.scan(/'(?:''|\\.|[^'\\])*'|"(?:""|\\.|[^"\\])*"|`(?:``|[^`])*`|--[^\n]*|\#[^\n]*|\/\*.*?\*\/|;|./m) do |token| + token = token.to_s + if token.start_with?("--", "#", "/*") + buffer << " " + elsif token == ";" + statements << buffer.strip unless buffer.strip.empty? + buffer = +"" + else + buffer << token + end + end + statements << buffer.strip unless buffer.strip.empty? + statements end end end diff --git a/ruby/migration/manifest.json b/ruby/migration/manifest.json index 100357042..75bf1d998 100644 --- a/ruby/migration/manifest.json +++ b/ruby/migration/manifest.json @@ -1,5 +1,21 @@ { "files": { + "mysql/main/001_create_river_migration.down.sql": "660d6b11eeb88589d3ce2053be165cda8df4c1d0172fbc1e8c3daa614e3f5b93", + "mysql/main/001_create_river_migration.up.sql": "e8821d3f7e89892d726fd94b278316e82835995cc6bf5def4641137de930f68c", + "mysql/main/002_initial_schema.down.sql": "93d74166cb838b13ae42447181afb2c2987dcb16ae33422de612daecb0163e79", + "mysql/main/002_initial_schema.up.sql": "dbe917102df1708a316b89c789a587d358fad11d6c8dac26f95a22e8691ad359", + "mysql/main/003_river_job_tags_non_null.down.sql": "dbfb0192d6254bdcf2a12c70c8b4ad44dfd7ebaa273b3160c0e76b9208f0a1fe", + "mysql/main/003_river_job_tags_non_null.up.sql": "e8a8459edd8e6110e564be74c03f646b430e2f3e31fef31eeb5610ed3b10660f", + "mysql/main/004_pending_and_more.down.sql": "f4325dd580c3725d7eb068262d56319f779b556ceddfb53771c51a855ca2440f", + "mysql/main/004_pending_and_more.up.sql": "21711fa26eeb7c3abb9cb440652ba4a79fac52bb6ae1e1e5204b59e8b5847718", + "mysql/main/005_migration_unique_client.down.sql": "2ba63bb2563f28f4a1fe1713c3336feb46dacf979079d0859682ec1e83b72cc5", + "mysql/main/005_migration_unique_client.up.sql": "a7634888e5e8f25121d8c75592451106bf7a2d7ab15c037b561d7650908b0688", + "mysql/main/006_bulk_unique.down.sql": "5c86b3bd5e32a1f8462b6e4f7ba8aa4fcb4c5be37f4c7d3c1faeb9602ca410fe", + "mysql/main/006_bulk_unique.up.sql": "f8625bc2fbff65b1b77fb416bc95889981256d890ce8d157bb1227064bff5228", + "mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql": "897217327df7fc1d85b57a3c785d70b4a33eae9c3957f7977b0a0365de826a17", + "mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql": "546a783aec00f2b2fdac03d2a902f4899c223d5789602091a31be166c3eebf18", + "mysql/main/008_job_id_autoincrement.down.sql": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa", + "mysql/main/008_job_id_autoincrement.up.sql": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa", "postgresql/main/001_create_river_migration.down.sql": "34c87dc594bf7520bc3ae69f6f0da8d2d9a472616ab38b37e63d4e3838da06d2", "postgresql/main/001_create_river_migration.up.sql": "79def9ab1643beee7776c499559ec199a03b5b26036c122dc3ba13ec3d078dc0", "postgresql/main/002_initial_schema.down.sql": "8e7e73755b3e9cd1d46f0dffeadd427b86af13cea2f41f3d30af1624329db9b9", diff --git a/ruby/migration/mysql/main/001_create_river_migration.down.sql b/ruby/migration/mysql/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/ruby/migration/mysql/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/ruby/migration/mysql/main/001_create_river_migration.up.sql b/ruby/migration/mysql/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..9aee5ff2f --- /dev/null +++ b/ruby/migration/mysql/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/ruby/migration/mysql/main/002_initial_schema.down.sql b/ruby/migration/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/ruby/migration/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/ruby/migration/mysql/main/002_initial_schema.up.sql b/ruby/migration/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..c680cb3db --- /dev/null +++ b/ruby/migration/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify is +-- omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB; diff --git a/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql b/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql b/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/ruby/migration/mysql/main/004_pending_and_more.down.sql b/ruby/migration/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/ruby/migration/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/ruby/migration/mysql/main/004_pending_and_more.up.sql b/ruby/migration/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..7725c9f12 --- /dev/null +++ b/ruby/migration/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. MySQL doesn't have enum types to +-- alter, so we update the CHECK constraint instead. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/ruby/migration/mysql/main/005_migration_unique_client.down.sql b/ruby/migration/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..108c807a8 --- /dev/null +++ b/ruby/migration/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/ruby/migration/mysql/main/005_migration_unique_client.up.sql b/ruby/migration/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..fc5889c45 --- /dev/null +++ b/ruby/migration/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; diff --git a/ruby/migration/mysql/main/006_bulk_unique.down.sql b/ruby/migration/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/ruby/migration/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/ruby/migration/mysql/main/006_bulk_unique.up.sql b/ruby/migration/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/ruby/migration/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..c4c740ea7 --- /dev/null +++ b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..d5dc9a6b4 --- /dev/null +++ b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql b/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql b/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/ruby/scripts/check_coverage.rb b/ruby/scripts/check_coverage.rb index 4d6137758..32fb38345 100644 --- a/ruby/scripts/check_coverage.rb +++ b/ruby/scripts/check_coverage.rb @@ -2,7 +2,7 @@ require "simplecov" -# Both backend jobs must supply every package's report; a missing artifact must +# All backend jobs must supply every package's report; a missing artifact must # not turn into an apparently complete report for only one package or backend. reports = %w[postgres sqlite].product([".", "driver/riverqueue-activerecord", "driver/riverqueue-sequel"]).map do |backend, package| path = "coverage/ci/ruby-coverage-#{backend}/#{package}/coverage/.resultset.json" @@ -10,6 +10,10 @@ path end +mysql_report = "coverage/ci/ruby-coverage-mysql/.resultset.json" +abort "Missing coverage report: #{mysql_report}" unless File.file?(mysql_report) +reports << mysql_report + SimpleCov.collate(reports) do enable_coverage :branch minimum_coverage branch: 100, line: 100 diff --git a/ruby/scripts/sync_migrations.rb b/ruby/scripts/sync_migrations.rb index 9e366eaba..7918557ee 100644 --- a/ruby/scripts/sync_migrations.rb +++ b/ruby/scripts/sync_migrations.rb @@ -11,7 +11,7 @@ root = File.expand_path("..", __dir__) destination = File.join(root, "migration") line = "main" -drivers = {"postgresql" => "riverdriver/riverpgxv5", "sqlite" => "riverdriver/riversqlite"} +drivers = {"mysql" => "riverdriver/rivermysql", "postgresql" => "riverdriver/riverpgxv5", "sqlite" => "riverdriver/riversqlite"} files = {} drivers.each do |backend, directory| diff --git a/ruby/sig/driver.rbs b/ruby/sig/driver.rbs index 00853c97a..753283f74 100644 --- a/ruby/sig/driver.rbs +++ b/ruby/sig/driver.rbs @@ -146,10 +146,12 @@ module River def runtime_job_list: (JobListParams, ?for_delete: bool) -> Array[JobRow] def runtime_job_rows: (String) -> untyped def runtime_json: (untyped) -> untyped + def runtime_locks_rows?: () -> bool def runtime_merge_metadata: (untyped) -> untyped def runtime_metadata_equals: (untyped, untyped) -> untyped def runtime_nullable_time: (untyped) -> untyped def runtime_notify: (String, untyped) -> untyped + def runtime_order_by: (JobListParams, String) -> String def sqlite_insert_nonces: (Array[JobInsertParams]) -> Array[String] def runtime_parse_json: (untyped) -> untyped def runtime_parse_time: (untyped) -> untyped diff --git a/ruby/sig/migrator.rbs b/ruby/sig/migrator.rbs index e283453b0..18083a469 100644 --- a/ruby/sig/migrator.rbs +++ b/ruby/sig/migrator.rbs @@ -37,7 +37,9 @@ module River def execute: (String) -> untyped def existing_versions: () -> Array[Integer] def line_column?: () -> bool + def mysql_statements: (String) -> Array[String] def query: (String) -> untyped + def schema_prefix: () -> String def session: [A] (?lock: bool) { () -> A } -> A def table: () -> String end diff --git a/ruby/sig/mysql_runtime.rbs b/ruby/sig/mysql_runtime.rbs new file mode 100644 index 000000000..ccc39a3e3 --- /dev/null +++ b/ruby/sig/mysql_runtime.rbs @@ -0,0 +1,16 @@ +module River::Driver + module MySQLRuntime + include Runtime + + def job_get_by_id: (untyped) -> untyped + def job_insert_many: (Array[JobInsertParams]) -> Array[[River::JobRow, bool]] + + private + + def mysql_columns: (String) -> String + def mysql_delete: (String, String, String, ?limit: untyped) -> untyped + def mysql_locked_job: (untyped) -> untyped + def mysql_queue_paused: (untyped, bool, untyped) -> untyped + def mysql_update: (String, String, String, Array[String]) -> untyped + end +end diff --git a/ruby/spec/conformance_spec.rb b/ruby/spec/conformance_spec.rb index 918c1b967..5d89e8ddb 100644 --- a/ruby/spec/conformance_spec.rb +++ b/ruby/spec/conformance_spec.rb @@ -2,19 +2,14 @@ require "spec_helper" -# The fixture-backed checks run once, alongside SQLite, without a Go setup in -# every Postgres matrix entry. -return unless RiverTestDatabase.enabled?(:sqlite) +# Keep the serverless SQLite checks and opt into MySQL with TEST_DATABASE. +return unless RiverTestDatabase.enabled?(:sqlite) || RiverTestDatabase.enabled?(:mysql) +require_relative "support/client_test_database" require_relative "support/conformance_fixtures" -require_relative "support/river_test_schema" require "riverqueue-activerecord" require "riverqueue-sequel" require "riverqueue/testing" -class ConformanceRecord < ActiveRecord::Base - self.abstract_class = true -end - RSpec.describe "Go-generated conformance fixtures" do fixture_source = RiverConformanceFixtures.read("unique_keys") # Preserve raw argument tokens, including duplicate keys and negative zero. @@ -131,24 +126,16 @@ class ConformanceRecord < ActiveRecord::Base expect(River::RESUMABLE_STEP_METADATA_KEY).to eq(keys.fetch("resumable_step")) end - # These exercise actual insertion/cancellation and the SQL outbox in both - # drivers without requiring a server. Postgres delivery/commit ordering is - # separately exercised by the shared driver specs. - %w[activerecord sequel].each do |adapter| - context "#{adapter} persisted protocol" do + # Exercise the same persisted protocol on both outbox backends. Postgres + # delivery/commit ordering is separately exercised by the shared driver specs. + %w[active_record sequel].product(%i[mysql sqlite]).each do |adapter, backend| + context "#{adapter} #{backend} persisted protocol", database: backend do around do |example| - if adapter == "activerecord" - ConformanceRecord.establish_connection(adapter: "sqlite3", database: ":memory:") - @driver = River::Driver::ActiveRecord.new(connection_class: ConformanceRecord) - else - database = Sequel.sqlite - @driver = River::Driver::Sequel.new(database) + ClientTestDatabase.public_send("with_#{adapter}", backend) do |driver| + @driver = driver + @client = River::Client.new(@driver) + example.run end - RiverTestSchema.load(@driver) - @client = River::Client.new(@driver) - example.run - ensure - (adapter == "activerecord") ? ConformanceRecord.remove_connection : database&.disconnect end # Ruby emits these messages; its polling runtime does not consume diff --git a/ruby/spec/driver_runtime_shared_examples.rb b/ruby/spec/driver_runtime_shared_examples.rb index 058f54ce4..b92d63a1b 100644 --- a/ruby/spec/driver_runtime_shared_examples.rb +++ b/ruby/spec/driver_runtime_shared_examples.rb @@ -15,6 +15,8 @@ def to_json = JSON.dump(value: @value) let(:now) { Time.utc(2026, 1, 2, 3, 4, 5) } it "rescues a job whose JSON rescue counter overflows a Ruby float" do + skip "MySQL rejects numbers outside the JSON numeric range" if driver.migration_backend == :mysql + inserted = client.insert(DriverRuntimeArgs.new(1), scheduled_at: now - 120).job encoded = driver.send(:runtime_json, '{"river:rescue_count":1e999}') driver.send(:runtime_execute, "UPDATE river_job SET metadata = #{encoded} WHERE id = #{inserted.id}") @@ -28,6 +30,8 @@ def to_json = JSON.dump(value: @value) end it "snoozes a job whose JSON snooze counter overflows a Ruby float" do + skip "MySQL rejects numbers outside the JSON numeric range" if driver.migration_backend == :mysql + worker = Object.new worker.define_singleton_method(:work) { |_job| raise River.job_snooze(60) } client = River::Client.new(driver, config: River::Config.new(workers: River::Workers.new.add("driver_runtime", worker))) @@ -176,7 +180,7 @@ def to_json = JSON.dump(value: @value) row = insert_job(1, scheduled_at: now - 1) driver.job_claim(id: row.id, attempted_by: "test", now: now) cancel_before_write = true - driver.define_singleton_method(:runtime_returning_ids) do |sql| + driver.define_singleton_method(:runtime_query_rows) do |sql| if cancel_before_write cancel_before_write = false job_cancel(row.id) @@ -246,7 +250,7 @@ def work(_job) row = insert_job(1, scheduled_at: now - 1) driver.job_claim(id: row.id, attempted_by: "test", now: now) cancel_before_write = true - driver.define_singleton_method(:runtime_returning_ids) do |sql| + driver.define_singleton_method(:runtime_query_rows) do |sql| if cancel_before_write cancel_before_write = false job_cancel(row.id) diff --git a/ruby/spec/driver_shared_examples.rb b/ruby/spec/driver_shared_examples.rb index b6cda7868..be95da3f1 100644 --- a/ruby/spec/driver_shared_examples.rb +++ b/ruby/spec/driver_shared_examples.rb @@ -154,9 +154,13 @@ class SimpleArgsWithInsertOpts < SimpleArgs it "does not clean up a finalized job retried after cleanup selected it" do job = client.insert(SimpleArgs.new(job_num: 1)).job client.job_update(job.id, River::JobUpdateParams.new(finalized_at: Time.now.utc - 120, state: River::JOB_STATE_COMPLETED)) + retried = false driver.define_singleton_method(:runtime_query_rows) do |sql| rows = super(sql) - job_retry(job.id) if sql.start_with?("SELECT id FROM river_job WHERE") + if !retried && sql.start_with?("SELECT id FROM river_job WHERE") + retried = true + job_retry(job.id) + end rows end @@ -567,13 +571,23 @@ class SimpleArgsWithInsertOpts < SimpleArgs stored = inserted.map do |result| driver.send(:runtime_query_rows, <<~SQL).first.values.first - SELECT #{postgres ? "args::text" : "json(args)"} AS args FROM river_job WHERE id = #{result.job.id} + SELECT #{if postgres + "args::text" + else + ((driver.migration_backend == :mysql) ? "CAST(args AS CHAR)" : "json(args)") + end} AS args FROM river_job WHERE id = #{result.job.id} SQL end - expect(stored).to all(eq(postgres ? + expected = postgres ? '{"decimal": 1.10, "unicode": "café", "exponent": 1000, "high_precision": 1234567890.1234567890}' : - encoded)) - expect(inserted.map { |result| result.job.args }).to all(eq(JSON.parse(encoded))) + encoded + if driver.migration_backend == :mysql + # MySQL normalizes its native JSON representation, including numbers. + literal = driver.send(:runtime_quote, encoded) + expected = driver.send(:runtime_query_rows, "SELECT CAST(CAST(#{literal} AS JSON) AS CHAR) AS args").first.values.first + end + expect(stored).to all(eq(expected)) + expect(inserted.map { |result| result.job.args }).to all(eq(JSON.parse(expected))) end end diff --git a/ruby/spec/migrator_shared_examples.rb b/ruby/spec/migrator_shared_examples.rb index 5e2e1c018..debb2864d 100644 --- a/ruby/spec/migrator_shared_examples.rb +++ b/ruby/spec/migrator_shared_examples.rb @@ -68,7 +68,11 @@ migrator.migrate(target: 6) @driver.migration_connection do |connection| sql = "INSERT INTO river_job (args, kind, max_attempts, metadata) VALUES ('{\"value\":42}', 'migration_test', 25, '{\"keep\":true}')" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end migrator.migrate @@ -105,13 +109,21 @@ migrator.migrate @driver.migration_connection do |connection| sql = "INSERT INTO river_migration (line, version) VALUES ('main', 9)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.migrate }.to raise_error(River::Error, /history/) @driver.migration_connection do |connection| sql = "DELETE FROM river_migration WHERE version IN (3, 9)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.status }.to raise_error(River::Error, /history/) @@ -121,7 +133,11 @@ migrator.migrate @driver.migration_connection do |connection| sql = "INSERT INTO river_migration (line, version) VALUES ('pro', 1)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.migrate(direction: :down, target: 4) }.to raise_error(River::Error, /non-main/) diff --git a/ruby/spec/migrator_spec.rb b/ruby/spec/migrator_spec.rb index 489a2562c..38ba32c23 100644 --- a/ruby/spec/migrator_spec.rb +++ b/ruby/spec/migrator_spec.rb @@ -29,7 +29,7 @@ expect { described_class.new(driver, schema: "other") }.to raise_error(ArgumentError, /SQLite/) end - [:postgres, :sqlite].each do |adapter| + [:mysql, :postgres, :sqlite].each do |adapter| context "with #{adapter}", database: adapter do around do |example| skip "Postgres unavailable" if adapter == :postgres && !DB @@ -57,6 +57,37 @@ expect { migrator.migrate }.to raise_error(River::Error, /changed concurrently/) end + if adapter == :mysql + it "supports explicit schemas and rejects unsafe identifiers" do + schema = @driver.send(:runtime_query_rows, "SELECT DATABASE() AS name").first.fetch(:name) + expect(described_class.new(@driver, schema: schema).migrate.length).to eq(8) + expect { described_class.new(@driver, schema: "bad'name").status }.to raise_error(ArgumentError, /identifier/) + end + + it "refuses concurrent migrators and releases its named lock" do + schema = @driver.send(:runtime_query_rows, "SELECT DATABASE() AS name").first.fetch(:name) + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL")) + admin.fetch("SELECT GET_LOCK('river_migrate:#{schema}', 0)").all + expect { described_class.new(@driver).migrate }.to raise_error(River::Error, /database lock/) + admin.fetch("SELECT RELEASE_LOCK('river_migrate:#{schema}')").all + expect(described_class.new(@driver).migrate.length).to eq(8) + ensure + admin&.disconnect + end + + it "requires a selected database" do + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL").sub(%r{/[^/]*$}, "/")) + expect { described_class.new(River::Driver::Sequel.new(admin)).status }.to raise_error(ArgumentError, /identifier/) + ensure + admin&.disconnect + end + + it "splits migration scripts while preserving quoted semicolons" do + migrator = described_class.new(@driver) + expect(migrator.send(:mysql_statements, "; SELECT 'a;b'; -- comment;\n SELECT \"x;y\"; /* ; */ SELECT `a;b` # comment;\n")).to eq(["SELECT 'a;b'", "SELECT \"x;y\"", "SELECT `a;b`"]) + end + end + if adapter == :postgres it "rejects an empty search path" do @driver.migration_connection do |connection| diff --git a/ruby/spec/mysql_spec.rb b/ruby/spec/mysql_spec.rb new file mode 100644 index 000000000..13d682122 --- /dev/null +++ b/ruby/spec/mysql_spec.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +require "spec_helper" +require "active_record" +require "sequel" +require "riverqueue-activerecord" +require "riverqueue-sequel" +require_relative "driver_shared_examples" +require_relative "support/client_test_database" + +RSpec.describe "MySQL drivers", database: :mysql do + [:active_record, :sequel].each do |adapter| + context adapter.to_s do + before(:all) do + @admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL", "mysql2://root@127.0.0.1/")) + @schema = "river_ruby_mysql_#{SecureRandom.hex(8)}" + @admin.run("CREATE DATABASE #{@schema}") + config = @admin.opts.merge(database: @schema, max_connections: 10) + if adapter == :sequel + @database = Sequel.connect(config) + @driver = River::Driver::Sequel.new(@database) + else + @original_connection = ClientTestDatabase.active_record_config + ActiveRecord::Base.establish_connection(adapter: "mysql2", host: config[:host], port: config[:port], + username: config[:user], password: config[:password], database: @schema, pool: 10) + @driver = River::Driver::ActiveRecord.new + end + River::Migrator.new(@driver).migrate + end + + after(:all) do + @database&.disconnect + if adapter == :active_record + if @original_connection + ActiveRecord::Base.establish_connection(@original_connection) + else + ActiveRecord::Base.remove_connection + end + end + @admin&.run("DROP DATABASE #{@schema}") if @schema + @admin&.disconnect + end + + around(:each) do |example| + driver.transaction do + example.run + raise driver.rollback_exception + end + end + + let(:driver) { (adapter == :sequel) ? River::Driver::Sequel.new(@database) : River::Driver::ActiveRecord.new } + let(:client) { River::Client.new(driver) } + + it_behaves_like "driver shared examples" + + it "handles empty batches and uses the database clock for an omitted schedule" do + expect(driver.job_insert_many([])).to eq([]) + params = River::Driver::JobInsertParams.new(encoded_args: "{}", kind: "simple", max_attempts: 25, + priority: 1, queue: "default", scheduled_at: nil, state: "available", tags: nil) + row, duplicate = driver.job_insert(params) + expect(duplicate).to be(false) + expect(row.scheduled_at).to be_within(2).of(Time.now.getutc) + expect { driver.send(:runtime_time, nil) }.to raise_error(ArgumentError, /nil/) + end + + it "appends scalar attempt errors without changing their JSON type" do + row = client.insert(SimpleArgs.new(job_num: 1)).job + sql = driver.send(:runtime_append_error, JSON.generate("a scalar error")) + driver.send(:runtime_execute, "UPDATE river_job SET errors = #{sql} WHERE id = #{row.id}") + expect(driver.job_get_by_id(row.id).errors).to contain_exactly(have_attributes(error: "a scalar error")) + end + + it "matches kinds and queues exactly, including trailing spaces" do + wanted = client.insert(SimpleArgs.new(job_num: 1)).job + padded = client.insert(SimpleArgs.new(job_num: 2)).job + driver.send(:runtime_execute, "UPDATE river_job SET kind = 'simple ', queue = 'default ' WHERE id = #{padded.id}") + expect(client.job_list(River::JobListParams.new(kinds: ["simple"])).jobs.map(&:id)).to eq([wanted.id]) + expect(client.job_list(River::JobListParams.new(queues: ["default"])).jobs.map(&:id)).to eq([wanted.id]) + expect(driver.job_get_available(queue: "default", max: 10, attempted_by: "test").map(&:id)).to eq([wanted.id]) + end + + it "keeps adjacent large IDs distinct" do + first = client.insert(SimpleArgs.new(job_num: 1)).job + second = client.insert(SimpleArgs.new(job_num: 2)).job + driver.send(:runtime_execute, "UPDATE river_job SET id = #{2**53} WHERE id = #{first.id}") + driver.send(:runtime_execute, "UPDATE river_job SET id = #{2**53 + 1} WHERE id = #{second.id}") + expect(client.job_list(River::JobListParams.new(ids: [2**53 + 1])).jobs.map(&:id)).to eq([2**53 + 1]) + end + end + end +end + +require_relative "client_driver_shared_examples" + +RSpec.describe "MySQL worker integration", database: :mysql do + [:active_record, :sequel].each do |adapter| + context adapter.to_s do + around do |example| + ClientTestDatabase.public_send("with_#{adapter}", :mysql) do |driver| + @driver = driver + example.run + end + end + + it_behaves_like "client driver end to end" + + it "rejects conflicts on an unrelated unique index" do + @driver.send(:runtime_execute, "CREATE UNIQUE INDEX mysql_test_kind ON river_job(kind)") + client = River::Client.new(@driver) + client.insert(SimpleArgs.new(job_num: 1)) + expect { client.insert(SimpleArgs.new(job_num: 2)) }.to raise_error(River::Error, /other than River/) + expect { client.insert(SimpleArgs.new(job_num: 3), unique_opts: River::UniqueOpts.new(by_args: true)) }.to raise_error(River::Error, /other than River/) + end + + it "rejects unrelated conflicts with inactive unique keys" do + @driver.send(:runtime_execute, "CREATE UNIQUE INDEX mysql_test_kind ON river_job(kind)") + client = River::Client.new(@driver) + unique = River::UniqueOpts.new(by_args: true, by_state: [:available, :pending, :running, :scheduled]) + params = client.send(:make_insert_params, SimpleArgs.new(job_num: 1), River::InsertOpts.new(unique_opts: unique)) + job, = @driver.job_insert(params) + params.state = "retryable" + expect { @driver.job_insert(params) }.to raise_error(River::Error, /other than River/) + + @driver.send(:runtime_execute, "UPDATE river_job SET unique_states = NULL WHERE id = #{job.id}") + params.state = "available" + expect { @driver.job_insert(params) }.to raise_error(River::Error, /other than River/) + end + + it "migrates down and back up using the Go migration history" do + migrator = River::Migrator.new(@driver) + expect(migrator.status).to all(have_attributes(applied: true)) + migrator.migrate(direction: :down, target: 0) + expect(migrator.status).to all(have_attributes(applied: false)) + migrator.migrate + expect(migrator.status).to all(have_attributes(applied: true)) + end + end + end +end + +RSpec.describe "MySQL cleaner", database: :mysql do + [:active_record, :sequel].each do |adapter| + it "skips locked jobs with #{adapter}" do + ClientTestDatabase.public_send("with_#{adapter}", :mysql) do |driver| + params = River::Driver::JobInsertParams.new(encoded_args: "{}", kind: "cleanup", max_attempts: 25, + priority: 1, queue: "default", scheduled_at: nil, state: "available", tags: []) + running, locked, *expired = 1002.times.map { driver.job_insert(params).first.id } + driver.send(:runtime_execute, "UPDATE river_job SET state = 'completed', finalized_at = '1970-01-01' WHERE id >= #{locked}") + driver.send(:runtime_execute, "UPDATE river_job SET state = 'running' WHERE id = #{running}") + ready = Queue.new + release = Queue.new + locker = Thread.new do + driver.transaction do + [running, locked].each do |id| + driver.send(:runtime_query_rows, "SELECT id FROM river_job WHERE id = #{id} FOR UPDATE") + end + ready << true + release.pop + end + end + begin + Timeout.timeout(5) { ready.pop } + deleted = Timeout.timeout(2) { driver.job_delete_finalized(retention: {completed: 0}, max: 1000) } + expect(deleted).to eq(1000) + expect(driver.job_get_by_id(expired.first)).to be_nil + expect(driver.job_get_by_id(locked).state).to eq("completed") + expect(driver.job_get_by_id(running).state).to eq("running") + ensure + release << true + locker.join + end + end + end + end +end diff --git a/ruby/spec/spec_helper.rb b/ruby/spec/spec_helper.rb index 6101182a6..c6f8f951e 100644 --- a/ruby/spec/spec_helper.rb +++ b/ruby/spec/spec_helper.rb @@ -14,9 +14,13 @@ minimum_coverage branch: 100, line: 100 if RiverTestDatabase::BACKEND == "all" # Drivers have their own spec suite where they're covered 100.0%, but - # they're not fully covered from this top level test suite. - add_filter("driver/riverqueue-sequel/") - add_filter("driver/riverqueue-activerecord/") + # they're not fully covered from this top level test suite. MySQL's + # adapter coverage is collected here alongside its shared driver tests. + unless RiverTestDatabase::BACKEND == "mysql" + add_filter("driver/riverqueue-sequel/") + add_filter("driver/riverqueue-activerecord/") + end + add_filter("/lib/driver/mysql_runtime.rb") unless RiverTestDatabase.enabled?(:mysql) add_filter("/spec/") end end diff --git a/ruby/spec/support/client_test_database.rb b/ruby/spec/support/client_test_database.rb index d7b58fedf..2b5424eba 100644 --- a/ruby/spec/support/client_test_database.rb +++ b/ruby/spec/support/client_test_database.rb @@ -8,8 +8,25 @@ # rollback-wrapped driver contracts. Never run their workers against public # tables: migrate an empty disposable schema with the bundled canonical SQL. module ClientTestDatabase + def self.active_record_config + ActiveRecord::Base.connection_handler.retrieve_connection_pool(ActiveRecord::Base.connection_specification_name)&.db_config&.configuration_hash + end + def self.with_active_record(adapter, migrate: true, pg_catalog_last: false) - original = ActiveRecord::Base.connection_db_config.configuration_hash + original = active_record_config + if adapter == :mysql + return with_mysql do |config| + ActiveRecord::Base.establish_connection(adapter: "mysql2", host: config[:host], port: config[:port], + username: config[:user], password: config[:password], database: config[:database], pool: 20) + begin + driver = River::Driver::ActiveRecord.new + RiverTestSchema.load(driver) if migrate + yield driver + ensure + ActiveRecord::Base.connection_pool.disconnect! + end + end + end if adapter == :postgres ActiveRecord::Base.establish_connection(ENV["TEST_DATABASE_URL"] || "postgres://localhost/river_test") schema = "river_client_test_#{SecureRandom.hex(8)}" @@ -42,11 +59,27 @@ def self.with_active_record(adapter, migrate: true, pg_catalog_last: false) begin ActiveRecord::Base.connection.execute("DROP SCHEMA #{schema} CASCADE") if schema_created ensure - ActiveRecord::Base.establish_connection(original) + if original + ActiveRecord::Base.establish_connection(original) + else + ActiveRecord::Base.remove_connection + end end end def self.with_sequel(adapter, migrate: true, pg_catalog_last: false) + if adapter == :mysql + return with_mysql do |config| + database = Sequel.connect(config.merge(max_connections: 20)) + begin + driver = River::Driver::Sequel.new(database) + RiverTestSchema.load(driver) if migrate + yield driver + ensure + database.disconnect + end + end + end if adapter == :postgres schema = "river_client_test_#{SecureRandom.hex(8)}" search_path = pg_catalog_last ? "#{schema},pg_catalog" : "#{schema},public" @@ -77,4 +110,14 @@ def self.with_sequel(adapter, migrate: true, pg_catalog_last: false) database&.disconnect end end + + def self.with_mysql + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL", "mysql2://root@127.0.0.1/")) + schema = "river_ruby_mysql_#{SecureRandom.hex(8)}" + admin.run("CREATE DATABASE #{schema}") + yield admin.opts.merge(database: schema) + ensure + admin&.run("DROP DATABASE #{schema}") if schema + admin&.disconnect + end end diff --git a/ruby/spec/support/migrations/mysql/test/001_test.down.sql b/ruby/spec/support/migrations/mysql/test/001_test.down.sql new file mode 100644 index 000000000..bfc26b8ca --- /dev/null +++ b/ruby/spec/support/migrations/mysql/test/001_test.down.sql @@ -0,0 +1 @@ +DROP TABLE river_migration_test; diff --git a/ruby/spec/support/migrations/mysql/test/001_test.up.sql b/ruby/spec/support/migrations/mysql/test/001_test.up.sql new file mode 100644 index 000000000..d985fb052 --- /dev/null +++ b/ruby/spec/support/migrations/mysql/test/001_test.up.sql @@ -0,0 +1 @@ +CREATE TABLE river_migration_test (id integer PRIMARY KEY); diff --git a/ruby/spec/support/river_test_schema.rb b/ruby/spec/support/river_test_schema.rb index 8a3388125..048f18163 100644 --- a/ruby/spec/support/river_test_schema.rb +++ b/ruby/spec/support/river_test_schema.rb @@ -5,6 +5,8 @@ # and advisory locks. Replay the same SQL, without maintaining a schema copy. module RiverTestSchema def self.load(driver) + return River::Migrator.new(driver).migrate if driver.migration_backend == :mysql + migrations = River::Migrator.new(driver).migrations sql = migrations.map do |migration| # Postgres must commit new enum values before later migrations use them. diff --git a/ruby/spec/support/test_database.rb b/ruby/spec/support/test_database.rb index b873d7417..207b4933e 100644 --- a/ruby/spec/support/test_database.rb +++ b/ruby/spec/support/test_database.rb @@ -3,14 +3,15 @@ # Like the Go suite, default to both backends and allow CI to test them separately. module RiverTestDatabase BACKEND = ENV.fetch("TEST_DATABASE", "all") - raise ArgumentError, "TEST_DATABASE must be all, postgres, or sqlite" unless %w[all postgres sqlite].include?(BACKEND) + raise ArgumentError, "TEST_DATABASE must be all, mysql, postgres, or sqlite" unless %w[all mysql postgres sqlite].include?(BACKEND) def self.enabled?(backend) + return BACKEND == "mysql" || (BACKEND == "all" && ENV.key?("TEST_MYSQL_URL")) if backend == :mysql + BACKEND == "all" || backend.to_s == BACKEND end end RSpec.configure do |config| - config.filter_run_excluding database: :postgres unless RiverTestDatabase.enabled?(:postgres) - config.filter_run_excluding database: :sqlite unless RiverTestDatabase.enabled?(:sqlite) + config.filter_run_excluding database: ->(backend) { backend && !RiverTestDatabase.enabled?(backend) } end diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 161da85e0..8bde5eae9 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -12,6 +12,8 @@ Changes to River for Go are recorded in the [repository changelog](../CHANGELOG. ### Added +- MySQL support through the `mysql` feature, including transactional job operations, workers, notification delivery, maintenance, test helpers, and CLI migrations using River for Go's canonical schema. + - A worker cancels its job with a reason by returning `JobCancelError`, like River for Go's `JobCancel(err)`. Anywhere in the worker error's source chain, it cancels the job whatever attempts it has left, skips the error handler, and records `JobCancelError: ` on the attempt as Go does. `WorkOutcome::Cancel` still cancels without a reason. [PR #1504](https://github.com/riverqueue/river/pull/1504). ### Changed diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c92c20acc..15b9318a3 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -949,6 +949,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "sqlx", + "sqlx-core", "thiserror", "tokio", "tokio-util", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c9fe3dd3d..0ef3c75a0 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,6 +29,7 @@ serde = { version = "1.0.229", features = ["derive"] } serde_json = { version = "1.0.151", features = ["raw_value"] } sha2 = "0.11.0" sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio"] } +sqlx-core = { version = "0.9.0", default-features = false } syn = { version = "3.0", features = ["full"] } thiserror = "2.0.20" tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] } @@ -40,7 +41,7 @@ missing_debug_implementations = "warn" unsafe_code = "forbid" # `--cfg river_postgres_tests` builds the Postgres integration tests, which # need `RIVER_RUST_DATABASE_URL`; `make test/rust` sets it when the URL is set. -unexpected_cfgs = { level = "warn", check-cfg = ["cfg(river_postgres_tests)"] } +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(river_postgres_tests)", "cfg(river_mysql_tests)"] } [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/rust/README.md b/rust/README.md index 4d4c79143..bc3212ad9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,6 +1,6 @@ # River for Rust -River is a fast, reliable background job system backed by Postgres or SQLite. Its Rust implementation uses Tokio and shares River's database schema and job protocol with the other languages, so services can insert and work jobs in the same database. +River is a fast, reliable background job system backed by Postgres, MySQL, or SQLite. Its Rust implementation uses Tokio and shares River's database schema and job protocol with the other languages, so services can insert and work jobs in the same database. ## Installation @@ -18,6 +18,7 @@ Rust version is 1.95. | Feature | Default | Enables | |---|---|---| +| `mysql` | no | MySQL 8.0 or newer through SQLx | | `postgres` | yes | Postgres through SQLx | | `sqlite` | no | SQLite 3.45 or newer through SQLx | | `chrono-tz` | no | IANA zone names such as `America/New_York` in cron `CRON_TZ=` and `TZ=` prefixes | @@ -35,6 +36,38 @@ compatible with River's (SQLx 0.9, Chrono 0.4, `serde_json` 1, and implementation for SQLx; enable one of SQLx's TLS features in your own SQLx dependency if your database connections use TLS. +## MySQL + +Enable the `mysql` feature, and pass a SQLx `MySqlPool` to the client: + +```toml +riverqueue = { version = "0.3.0", default-features = false, features = ["mysql"] } +``` + +```rust,no_run +# #[cfg(feature = "mysql")] +# async fn example() -> Result<(), Box> { +use riverqueue::{Client, migrate::MySqlMigrator, sqlx::MySqlPool}; + +let pool = MySqlPool::connect("mysql://localhost/app").await?; +MySqlMigrator::new(pool.clone()).migrate_up().await?; +let client = Client::builder(pool).build()?; +# Ok(()) +# } +``` + +Create the database before migrating. Jobs, workers, queues, and `.tx(...)` +requests use the same APIs as the other backends. MySQL clients poll River's +transactional notification outbox for insertions, queue changes, cancellation, +and leadership requests. River's own transactions use READ COMMITTED isolation; +caller-owned transactions retain their configured isolation. Use +`riverqueue::database::begin_mysql_write` for a cancellation-safe begin at +READ COMMITTED. + +MySQL implicitly commits DDL, so a failed migration may leave successful +statements applied. Migration history advances only after all statements in +that migration succeed. + ## Quick start Set `DATABASE_URL` to your application's Postgres database. Define serializable job arguments, register a worker, apply migrations, and start a client: @@ -105,7 +138,7 @@ Insert jobs in the same transaction as application data by chaining `.tx(&mut tr - [Unique, periodic, and resumable jobs](riverqueue/README.md#reliability-features). - [Event subscriptions](riverqueue/README.md#events) for logging and metrics. - [Leader election and maintenance](riverqueue/README.md#leadership-and-maintenance) for scheduling, rescue, and cleanup. -- [Postgres and SQLite support](riverqueue/README.md#database-support) with caller-owned SQLx pools. +- [MySQL, Postgres, and SQLite support](riverqueue/README.md#database-support) with caller-owned SQLx pools. ## Crates diff --git a/rust/docs/development.md b/rust/docs/development.md index 7e1e971ae..fbf19cf78 100644 --- a/rust/docs/development.md +++ b/rust/docs/development.md @@ -32,6 +32,15 @@ committed. The `make test/rust` targets generate them first, so Go is needed to run the tests; when running `cargo test` directly, run `make generate/fixtures` beforehand. A missing fixture fails its test. +`RIVER_RUST_MYSQL_URL=mysql://root@localhost/ make test/rust/conformance/mysql` +runs the Go fixture checks against MySQL storage: states and unique bits, +attempt errors, rescue/output metadata, snooze counters through real workers, +and notification emission and reception, including rollback. Tests create and +drop their own databases using the canonical migrations. The shared Conformance +workflow runs these checks when Go changes, and `make test/rust/mysql` includes +them in the full MySQL suite. Like Go's MySQL driver, Rust does not emit a +`resigned` notification; the fixture suite checks reception of that message. + `make check/rust/package` builds the five publishable crate archives and verifies that each one builds from its packaged sources, resolving the exact-version workspace dependencies from the other archives. It does not @@ -102,3 +111,16 @@ Run these commands from the repository root. All five Rust crates are versioned ``` 7. Create a [GitHub release](https://github.com/riverqueue/river/releases/new), select `rust/v$VERSION`, and copy the version's `rust/CHANGELOG.md` content into the release body. Mark alpha, beta, and RC versions as prereleases. + +### MySQL + +Run MySQL integration tests with a URL whose user can create temporary databases: + +```sh +RIVER_RUST_MYSQL_URL=mysql://root@localhost/ make test/rust/mysql +``` + +The test runner enables `river_mysql_tests` explicitly, so a missing database +URL fails the target. CI exercises MySQL 8.0 and 8.4. MySQL migration mirrors +come from `riverdriver/rivermysql/migration/main` and are regenerated by +`make generate/rust-migrations`. diff --git a/rust/riverqueue-cli/Cargo.toml b/rust/riverqueue-cli/Cargo.toml index fa22c9c97..0c6d621e0 100644 --- a/rust/riverqueue-cli/Cargo.toml +++ b/rust/riverqueue-cli/Cargo.toml @@ -15,8 +15,9 @@ homepage.workspace = true include = ["src/**", "LICENSE", "README.md"] [features] -default = ["postgres", "sqlite"] +default = ["mysql", "postgres", "sqlite"] postgres = ["riverqueue/postgres", "riverqueue-migrate/postgres", "sqlx/postgres"] +mysql = ["riverqueue/mysql", "riverqueue-migrate/mysql", "sqlx/mysql"] sqlite = ["riverqueue/sqlite", "riverqueue-migrate/sqlite", "sqlx/sqlite"] [[bin]] diff --git a/rust/riverqueue-cli/README.md b/rust/riverqueue-cli/README.md index 1a27fb7ec..fdcd9a8f4 100644 --- a/rust/riverqueue-cli/README.md +++ b/rust/riverqueue-cli/README.md @@ -16,8 +16,12 @@ implementation. Apply them before starting clients: riverqueue migrate-up --database-url postgres://localhost/app riverqueue migrate-up --database-url postgres://localhost/app --schema river riverqueue migrate-up --database-url sqlite://app.sqlite3 +riverqueue migrate-up --database-url mysql://localhost/app ``` +The CLI includes Postgres, MySQL, and SQLite support by default. For MySQL, +select an existing database in the URL; `--schema` is a Postgres option. + `migrate-down`, `migrate-list`, and `validate` take the same connection options. `--target-version N`, `--max-steps N`, and `--dry-run` limit or preview a migration run. Applications can instead migrate from Rust with the diff --git a/rust/riverqueue-cli/src/migrate.rs b/rust/riverqueue-cli/src/migrate.rs index 07f11ee8c..51a619f16 100644 --- a/rust/riverqueue-cli/src/migrate.rs +++ b/rust/riverqueue-cli/src/migrate.rs @@ -4,11 +4,15 @@ use std::error::Error; #[cfg(feature = "postgres")] use riverqueue::database::SchemaName; +#[cfg(feature = "mysql")] +use riverqueue_migrate::MySqlMigrator; #[cfg(feature = "postgres")] use riverqueue_migrate::PostgresMigrator; #[cfg(feature = "sqlite")] use riverqueue_migrate::SqliteMigrator; use riverqueue_migrate::{Direction, MigrateOpts, MigrateResult, ValidateResult}; +#[cfg(feature = "mysql")] +use sqlx::MySqlPool; #[cfg(feature = "postgres")] use sqlx::PgPool; #[cfg(feature = "sqlite")] @@ -27,6 +31,8 @@ struct Args { enum CommandMigrator { #[cfg(feature = "postgres")] Postgres(PostgresMigrator), + #[cfg(feature = "mysql")] + MySql(MySqlMigrator), #[cfg(feature = "sqlite")] Sqlite(SqliteMigrator), } @@ -36,6 +42,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.existing_versions().await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.existing_versions().await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.existing_versions().await, } @@ -49,6 +57,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.migrate(direction, opts).await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.migrate(direction, opts).await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.migrate(direction, opts).await, } @@ -61,6 +71,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.validate(target_version).await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.validate(target_version).await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.validate(target_version).await, } @@ -77,7 +89,19 @@ pub(crate) async fn run( println!("{}", usage()); return Ok(()); }; - let migrator = if args.database_url.starts_with("sqlite:") { + let migrator = if args.database_url.starts_with("mysql:") { + #[cfg(not(feature = "mysql"))] + return Err("MySQL support requires the `mysql` feature".into()); + #[cfg(feature = "mysql")] + { + if args.schema.is_some() { + return Err("select the MySQL database in --database-url".into()); + } + CommandMigrator::MySql(MySqlMigrator::new( + MySqlPool::connect(&args.database_url).await?, + )) + } + } else if args.database_url.starts_with("sqlite:") { #[cfg(not(feature = "sqlite"))] return Err("SQLite support requires the `sqlite` feature".into()); #[cfg(feature = "sqlite")] diff --git a/rust/riverqueue-migrate/Cargo.toml b/rust/riverqueue-migrate/Cargo.toml index 3bfe0a1cf..8accfcf83 100644 --- a/rust/riverqueue-migrate/Cargo.toml +++ b/rust/riverqueue-migrate/Cargo.toml @@ -20,6 +20,7 @@ all-features = true [features] default = ["postgres"] postgres = ["sqlx/postgres"] +mysql = ["sqlx/mysql", "sqlx/tls-rustls"] sqlite = ["sqlx/sqlite"] [dependencies] diff --git a/rust/riverqueue-migrate/README.md b/rust/riverqueue-migrate/README.md index 6441972f6..471fcdeb2 100644 --- a/rust/riverqueue-migrate/README.md +++ b/rust/riverqueue-migrate/README.md @@ -1,9 +1,9 @@ # riverqueue-migrate -River's Postgres and SQLite migrations for Rust, identical to the ones other -River clients apply, so a client in any language can migrate a database the -others use. The `riverqueue` command from the `riverqueue-cli` crate runs the -same migrations from a shell. +River's Postgres, MySQL, and SQLite migrations for Rust, identical to the ones +other River clients apply, so a client in any language can migrate a database +the others use. The `riverqueue` command from the `riverqueue-cli` crate runs +the same migrations from a shell. Use `PostgresMigrator` for Postgres and `SqliteMigrator` for SQLite: @@ -33,3 +33,10 @@ async fn migrate(postgres: PgPool, sqlite: SqlitePool) -> Result<(), Box= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..c680cb3db --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,43 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify is +-- omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..7725c9f12 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. MySQL doesn't have enum types to +-- alter, so we update the CHECK constraint instead. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..108c807a8 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..fc5889c45 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..c4c740ea7 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..d5dc9a6b4 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/rust/riverqueue-migrate/src/lib.rs b/rust/riverqueue-migrate/src/lib.rs index 912431fa8..6408ebc1a 100644 --- a/rust/riverqueue-migrate/src/lib.rs +++ b/rust/riverqueue-migrate/src/lib.rs @@ -1,4 +1,4 @@ -// The README's example uses both backends, so it's the crate documentation +// The README's Postgres and SQLite example needs both features, so it's the crate documentation // (and its example runs as a doctest) when both are enabled, as on docs.rs. #![cfg_attr( all(feature = "postgres", feature = "sqlite"), @@ -6,7 +6,7 @@ )] #![cfg_attr( not(all(feature = "postgres", feature = "sqlite")), - doc = "River's Postgres and SQLite migrations for Rust. The full crate documentation, \ + doc = "River's MySQL, Postgres, and SQLite migrations for Rust. The full crate documentation, \ from the README, builds with both the `postgres` and `sqlite` features, as on \ [docs.rs](https://docs.rs/riverqueue-migrate)." )] @@ -14,8 +14,10 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -#[cfg(not(any(feature = "postgres", feature = "sqlite")))] -compile_error!("riverqueue-migrate requires at least one database feature: `postgres` or `sqlite`"); +#[cfg(not(any(feature = "postgres", feature = "sqlite", feature = "mysql")))] +compile_error!( + "riverqueue-migrate requires at least one database feature: `mysql`, `postgres`, or `sqlite`" +); use std::time::Duration; #[cfg(feature = "postgres")] @@ -25,7 +27,11 @@ use std::time::Instant; use sqlx::{PgPool, Row}; use thiserror::Error; +#[cfg(feature = "mysql")] +mod mysql; mod schema; +#[cfg(feature = "mysql")] +pub use mysql::{MYSQL_MIGRATIONS, MySqlMigrator}; #[cfg(feature = "sqlite")] mod sqlite; @@ -484,6 +490,7 @@ impl PostgresMigrator { /// Runs a migration step on its own task and waits for it, so the step /// finishes even when the caller stops waiting. +#[cfg(any(feature = "postgres", feature = "sqlite"))] async fn run_to_completion( step: impl std::future::Future> + Send + 'static, ) -> Result<(), Error> { diff --git a/rust/riverqueue-migrate/src/mysql.rs b/rust/riverqueue-migrate/src/mysql.rs new file mode 100644 index 000000000..1bedf897f --- /dev/null +++ b/rust/riverqueue-migrate/src/mysql.rs @@ -0,0 +1,264 @@ +use std::time::{Duration, Instant}; + +use sqlx::{MySqlConnection, MySqlPool}; + +use crate::{ + Direction, Error, MIGRATION_LINE_MAIN, MigrateOpts, MigrateResult, MigrateVersion, Migration, + ValidateResult, select_migrations, validate_migrations, validate_target, +}; + +macro_rules! mysql_migration { + ($version:literal, $name:literal, $file:literal) => { + Migration { + down_sql: include_str!(concat!("../migrations/mysql/main/", $file, ".down.sql")), + name: $name, + up_sql: include_str!(concat!("../migrations/mysql/main/", $file, ".up.sql")), + version: $version, + } + }; +} + +/// Canonical MySQL migration bundle, shared with River's other clients. +pub const MYSQL_MIGRATIONS: [Migration; 8] = [ + mysql_migration!(1, "create_river_migration", "001_create_river_migration"), + mysql_migration!(2, "initial_schema", "002_initial_schema"), + mysql_migration!(3, "river_job_tags_non_null", "003_river_job_tags_non_null"), + mysql_migration!(4, "pending_and_more", "004_pending_and_more"), + mysql_migration!(5, "migration_unique_client", "005_migration_unique_client"), + mysql_migration!(6, "bulk_unique", "006_bulk_unique"), + mysql_migration!( + 7, + "notification_outbox_sqlite_jsonb_and_sql_cleanup", + "007_notification_outbox_sqlite_jsonb_and_sql_cleanup" + ), + mysql_migration!(8, "job_id_autoincrement", "008_job_id_autoincrement"), +]; + +/// Applies migrations to the MySQL database selected by the pool. +/// +/// The database must already exist. MySQL implicitly commits DDL: successful +/// statements remain applied after an error. A named lock serializes migrators, +/// and a version is recorded only after all its statements succeed. +#[derive(Clone, Debug)] +pub struct MySqlMigrator { + pool: MySqlPool, +} + +impl MySqlMigrator { + /// Creates a migrator for the pool's database. + #[must_use] + pub const fn new(pool: MySqlPool) -> Self { + Self { pool } + } + + /// Returns every bundled MySQL migration. + #[must_use] + pub fn all_versions() -> &'static [Migration] { + &MYSQL_MIGRATIONS + } + + /// Returns applied main-line versions in ascending order. + /// + /// # Errors + /// Returns the database error if the history cannot be read. + pub async fn existing_versions(&self) -> Result, Error> { + existing_versions(&mut *self.pool.acquire().await?).await + } + + /// Applies migrations with target, step, and dry-run controls. + /// + /// # Errors + /// Returns invalid-target errors or database errors, including failure to + /// acquire the migration lock within ten seconds. Other migration lines + /// prevent reverting version 5. + /// + /// # Cancel safety + /// Migration runs on its own task. Dropping this future lets the selected + /// migrations finish and releases the named lock before returning the + /// connection to its pool. + /// + /// # Panics + /// Resumes a panic from the migration task. + pub async fn migrate( + &self, + direction: Direction, + opts: MigrateOpts, + ) -> Result { + validate_target(&MYSQL_MIGRATIONS, opts.target_version, true)?; + let migrator = self.clone(); + match tokio::spawn(async move { migrator.migrate_locked(direction, opts).await }).await { + Ok(result) => result, + Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), + Err(error) => Err(sqlx::Error::Io(std::io::Error::other(format!( + "migration task failed: {error}" + ))) + .into()), + } + } + + /// Applies all outstanding migrations and returns their versions. + /// + /// # Errors + /// Returns the database error if a migration fails. + pub async fn migrate_up(&self) -> Result, Error> { + Ok(self + .migrate(Direction::Up, MigrateOpts::default()) + .await? + .versions + .into_iter() + .map(|version| version.version) + .collect()) + } + + /// Checks that every migration through the optional target is applied. + /// + /// # Errors + /// Returns invalid-target errors or errors reading migration history. + pub async fn validate(&self, target_version: Option) -> Result { + validate_target(&MYSQL_MIGRATIONS, target_version, false)?; + let applied = self.existing_versions().await?; + Ok(validate_migrations( + &MYSQL_MIGRATIONS, + target_version, + &applied, + )) + } + + async fn migrate_locked( + &self, + direction: Direction, + opts: MigrateOpts, + ) -> Result { + let mut connection = self.pool.acquire().await?; + let acquired: Option = + sqlx::query_scalar("SELECT GET_LOCK(CONCAT('river_migrate:', DATABASE()), 10)") + .fetch_one(&mut *connection) + .await?; + if acquired != Some(1) { + return Err(sqlx::Error::Protocol( + "could not acquire MySQL migration lock; select a database in the connection URL" + .into(), + ) + .into()); + } + // A task abort or panic closes the connection, releasing its named lock. + connection.close_on_drop(); + let result = migrate_on(&mut connection, direction, opts).await; + let released = sqlx::query("SELECT RELEASE_LOCK(CONCAT('river_migrate:', DATABASE()))") + .execute(&mut *connection) + .await; + match result { + Ok(result) => { + released?; + Ok(result) + } + Err(error) => Err(error), + } + } +} + +async fn existing_versions(connection: &mut MySqlConnection) -> Result, Error> { + let exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = 'river_migration')").fetch_one(&mut *connection).await?; + if !exists { + return Ok(Vec::new()); + } + let has_line: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = 'river_migration' AND column_name = 'line')").fetch_one(&mut *connection).await?; + Ok(if has_line { + sqlx::query_scalar( + "SELECT CAST(version AS SIGNED) FROM river_migration WHERE line = ? ORDER BY version", + ) + .bind(MIGRATION_LINE_MAIN) + .fetch_all(connection) + .await? + } else { + sqlx::query_scalar("SELECT CAST(version AS SIGNED) FROM river_migration ORDER BY version") + .fetch_all(connection) + .await? + }) +} + +async fn migrate_on( + connection: &mut MySqlConnection, + direction: Direction, + opts: MigrateOpts, +) -> Result { + let applied = existing_versions(connection).await?; + let selected = select_migrations(&MYSQL_MIGRATIONS, direction, opts, &applied)?; + let mut versions = Vec::with_capacity(selected.len()); + for migration in selected { + let sql = match direction { + Direction::Down => migration.down_sql, + Direction::Up => migration.up_sql, + } + .replace("/* TEMPLATE: schema */", ""); + let mut duration = Duration::ZERO; + if !opts.dry_run { + let start = Instant::now(); + apply(connection, direction, migration, &sql).await?; + duration = start.elapsed(); + } + versions.push(MigrateVersion { + duration, + name: migration.name, + sql, + version: migration.version, + }); + } + Ok(MigrateResult { + direction, + versions, + }) +} + +async fn apply( + connection: &mut MySqlConnection, + direction: Direction, + migration: Migration, + sql: &str, +) -> Result<(), Error> { + if direction == Direction::Down && migration.version == 5 { + let other_lines: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM river_migration WHERE line <> ?)") + .bind(MIGRATION_LINE_MAIN) + .fetch_one(&mut *connection) + .await?; + if other_lines { + return Err(Error::OtherMigrationLines { + version: migration.version, + }); + } + } + sqlx::raw_sql(sqlx::AssertSqlSafe(sql)) + .execute(&mut *connection) + .await?; + match direction { + Direction::Down if migration.version == 1 => {} + Direction::Down if migration.version <= 5 => { + sqlx::query("DELETE FROM river_migration WHERE version = ?") + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Down => { + sqlx::query("DELETE FROM river_migration WHERE line = ? AND version = ?") + .bind(MIGRATION_LINE_MAIN) + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Up if migration.version >= 5 => { + sqlx::query("INSERT INTO river_migration (line, version) VALUES (?, ?)") + .bind(MIGRATION_LINE_MAIN) + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Up => { + sqlx::query("INSERT INTO river_migration (version) VALUES (?)") + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + } + Ok(()) +} diff --git a/rust/riverqueue-test/Cargo.toml b/rust/riverqueue-test/Cargo.toml index c4cd207f8..66eb5e871 100644 --- a/rust/riverqueue-test/Cargo.toml +++ b/rust/riverqueue-test/Cargo.toml @@ -20,6 +20,7 @@ all-features = true [features] default = ["postgres"] postgres = ["riverqueue/postgres"] +mysql = ["riverqueue/mysql"] sqlite = ["riverqueue/sqlite"] [dependencies] diff --git a/rust/riverqueue/Cargo.toml b/rust/riverqueue/Cargo.toml index 6a3d33c1a..ba8ae8332 100644 --- a/rust/riverqueue/Cargo.toml +++ b/rust/riverqueue/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true homepage.workspace = true # Only what builds and documents the crate ships; tests and their fixtures stay # in the repository. -include = ["src/**", "docs/**", "examples/**", "LICENSE", "README.md"] +include = ["src/**", "!src/**/tests/**", "docs/**", "examples/**", "LICENSE", "README.md"] [package.metadata.docs.rs] all-features = true @@ -23,6 +23,7 @@ default = ["postgres"] # prefixes from a bundled time zone database. chrono-tz = ["dep:chrono-tz"] postgres = ["riverqueue-migrate/postgres", "sqlx/postgres"] +mysql = ["riverqueue-migrate/mysql", "sqlx/mysql", "sqlx/tls-rustls", "dep:sqlx-core"] sqlite = ["riverqueue-migrate/sqlite", "sqlx/sqlite"] [dependencies] @@ -38,6 +39,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlx = { workspace = true, features = ["chrono", "derive", "json"] } +sqlx-core = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/rust/riverqueue/README.md b/rust/riverqueue/README.md index ec4cd8c78..f8f2a352b 100644 --- a/rust/riverqueue/README.md +++ b/rust/riverqueue/README.md @@ -1,7 +1,7 @@ # riverqueue `riverqueue` is the Rust and Tokio client for [River](https://riverqueue.com), -a fast and reliable background job system backed by Postgres or SQLite. It +a fast and reliable background job system backed by Postgres, MySQL, or SQLite. It shares River's database schema and job protocol with River clients in other languages, so services written in different languages can insert and work jobs in the same database. @@ -24,6 +24,7 @@ Rust version is 1.95. | Feature | Default | Enables | |---|---|---| +| `mysql` | no | MySQL 8.0 or newer through SQLx | | `postgres` | yes | Postgres through SQLx | | `sqlite` | no | SQLite 3.45 or newer through SQLx | | `chrono-tz` | no | IANA zone names such as `America/New_York` in cron `CRON_TZ=` and `TZ=` prefixes | @@ -41,6 +42,41 @@ compatible with River's (SQLx 0.9, Chrono 0.4, `serde_json` 1, and implementation for SQLx; enable one of SQLx's TLS features in your own SQLx dependency if your database connections use TLS. +## MySQL + +Enable the `mysql` feature, and pass a SQLx `MySqlPool` to the client: + +```toml +riverqueue = { version = "0.3.0", default-features = false, features = ["mysql"] } +``` + +The MySQL feature includes SQLx's Rustls transport for password authentication +over TLS. + +```rust,no_run +# #[cfg(feature = "mysql")] +# async fn example() -> Result<(), Box> { +use riverqueue::{Client, migrate::MySqlMigrator, sqlx::MySqlPool}; + +let pool = MySqlPool::connect("mysql://localhost/app").await?; +MySqlMigrator::new(pool.clone()).migrate_up().await?; +let client = Client::builder(pool).build()?; +# Ok(()) +# } +``` + +Create the database before migrating. Jobs, workers, queues, and `.tx(...)` +requests use the same APIs as the other backends. MySQL clients poll River's +transactional notification outbox for insertions, queue changes, cancellation, +and leadership requests. River's own transactions use READ COMMITTED isolation; +caller-owned transactions retain their configured isolation. Use +`riverqueue::database::begin_mysql_write` for a cancellation-safe begin at +READ COMMITTED. + +MySQL implicitly commits DDL, so a failed migration may leave successful +statements applied. Migration history advances only after all statements in +that migration succeed. + ## Quick start Define serializable arguments, register an async function or a [`Worker`], @@ -333,7 +369,7 @@ async fn sqlite_client() -> Result> { - [`queue`] and [`query`]: queue records and job list filters and cursors. - [`periodic`]: schedules and runtime periodic job registration. - [`extension`]: hooks, middleware, policies, and metrics. -- [`database`]: Postgres and SQLite database options, and the transactions +- [`database`]: MySQL, Postgres, and SQLite database options, and the transactions River's `.tx` methods accept. - [`error`]: structured errors that keep their sources. - [`protocol`]: wire values such as notification topics and unique keys, for diff --git a/rust/riverqueue/src/__private.rs b/rust/riverqueue/src/__private.rs index 50d18e6ac..678dcd6de 100644 --- a/rust/riverqueue/src/__private.rs +++ b/rust/riverqueue/src/__private.rs @@ -15,6 +15,8 @@ use serde_json::{Map, Value, value::RawValue}; #[cfg(feature = "postgres")] use crate::database::SchemaName; use crate::{AttemptError, InsertResult, Job, JobRow, JobState}; +#[cfg(feature = "mysql")] +use sqlx::{MySqlConnection, MySqlPool}; #[cfg(feature = "postgres")] use sqlx::{PgConnection, PgPool}; #[cfg(feature = "sqlite")] @@ -150,6 +152,15 @@ pub fn attempt_errors_from_json(json: &str) -> Result, serde_j AttemptError::from_json_array_lenient(json) } +/// Encodes a UTC timestamp in River's canonical MySQL wire format. +/// +/// This keeps companion crates aligned with River and Go's +/// microsecond-precision, timezone-free MySQL representation. +#[cfg(feature = "mysql")] +#[must_use] +pub fn mysql_timestamp(time: DateTime) -> String { + crate::database::mysql::mysql_time(time) +} /// Encodes a UTC timestamp in River's canonical SQLite wire format. /// /// This keeps companion crates aligned with River and Go's @@ -330,6 +341,17 @@ pub async fn notify_many( .await?; Ok(()) } + #[cfg(feature = "mysql")] + (DatabaseConnection::MySql(connection), DatabaseConfig::MySql) => { + let mut query = sqlx::QueryBuilder::::new( + "INSERT INTO river_notification (payload, topic) ", + ); + query.push_values(payloads, |mut row, payload| { + row.push_bind(payload).push_bind(topic.as_str()); + }); + query.build().execute(connection).await?; + Ok(()) + } #[cfg(feature = "sqlite")] (DatabaseConnection::Sqlite(connection), DatabaseConfig::Sqlite) => { let mut query = sqlx::QueryBuilder::::new( @@ -407,6 +429,10 @@ pub async fn delete_finalized_jobs( (DatabaseConnection::Postgres(connection), DatabaseConfig::Postgres { schema }) => Ok( crate::maintenance::postgres_delete_finalized_jobs(connection, schema, params).await?, ), + #[cfg(feature = "mysql")] + (DatabaseConnection::MySql(connection), DatabaseConfig::MySql) => { + Ok(crate::maintenance::mysql_delete_finalized_jobs(connection, params).await?) + } #[cfg(feature = "sqlite")] (DatabaseConnection::Sqlite(connection), DatabaseConfig::Sqlite) => { Ok(crate::maintenance::sqlite_delete_finalized_jobs(connection, params).await?) @@ -434,6 +460,9 @@ pub enum DatabaseConfig { /// Postgres backend configuration. #[cfg(feature = "postgres")] Postgres { schema: SchemaName }, + /// MySQL backend configuration. + #[cfg(feature = "mysql")] + MySql, /// SQLite backend configuration. #[cfg(feature = "sqlite")] Sqlite, @@ -446,6 +475,8 @@ impl DatabaseConfig { match self { #[cfg(feature = "postgres")] Self::Postgres { .. } => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite => DatabaseKind::Sqlite, } @@ -457,6 +488,8 @@ impl DatabaseConfig { pub const fn postgres_schema(&self) -> Option<&SchemaName> { match self { Self::Postgres { schema } => Some(schema), + #[cfg(feature = "mysql")] + Self::MySql => None, #[cfg(feature = "sqlite")] Self::Sqlite => None, } @@ -470,6 +503,9 @@ pub enum DatabaseConnection<'connection> { /// Postgres transaction connection. #[cfg(feature = "postgres")] Postgres(&'connection mut PgConnection), + /// MySQL transaction connection. + #[cfg(feature = "mysql")] + MySql(&'connection mut MySqlConnection), /// SQLite transaction connection. #[cfg(feature = "sqlite")] Sqlite(&'connection mut SqliteConnection), @@ -482,6 +518,8 @@ impl<'connection> DatabaseConnection<'connection> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -494,6 +532,8 @@ impl<'connection> DatabaseConnection<'connection> { match self { #[cfg(feature = "postgres")] Self::Postgres(connection) => DatabaseConnection::Postgres(connection), + #[cfg(feature = "mysql")] + Self::MySql(connection) => DatabaseConnection::MySql(connection), #[cfg(feature = "sqlite")] Self::Sqlite(connection) => DatabaseConnection::Sqlite(connection), } @@ -505,6 +545,8 @@ impl<'connection> DatabaseConnection<'connection> { pub fn into_postgres(self) -> Option<&'connection mut PgConnection> { match self { Self::Postgres(connection) => Some(connection), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } @@ -512,12 +554,24 @@ impl<'connection> DatabaseConnection<'connection> { /// Returns the SQLite connection, if selected. #[must_use] + #[cfg(feature = "mysql")] + pub fn into_mysql(self) -> Option<&'connection mut MySqlConnection> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(connection) => Some(connection), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } #[cfg(feature = "sqlite")] pub fn into_sqlite(self) -> Option<&'connection mut SqliteConnection> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(connection) => Some(connection), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } } @@ -539,6 +593,9 @@ pub enum DatabasePool { /// Postgres pool. #[cfg(feature = "postgres")] Postgres(PgPool), + /// MySQL pool. + #[cfg(feature = "mysql")] + MySql(MySqlPool), /// SQLite pool. #[cfg(feature = "sqlite")] Sqlite(SqlitePool), @@ -551,6 +608,8 @@ impl DatabasePool { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -562,6 +621,8 @@ impl DatabasePool { pub const fn postgres(&self) -> Option<&PgPool> { match self { Self::Postgres(pool) => Some(pool), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } @@ -569,12 +630,24 @@ impl DatabasePool { /// Returns the caller-owned SQLite pool, if selected. #[must_use] + #[cfg(feature = "mysql")] + pub const fn mysql(&self) -> Option<&MySqlPool> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(pool) => Some(pool), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } #[cfg(feature = "sqlite")] pub const fn sqlite(&self) -> Option<&SqlitePool> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(pool) => Some(pool), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } } @@ -1177,11 +1250,25 @@ pub fn claimed_postgres_job(row: &sqlx::postgres::PgRow) -> ClaimedJob { ClaimedJob::from_decoded(crate::client::decode_job_row(row)) } +/// Columns River selects to decode a MySQL job row, for use with +/// [`decode_mysql_job_row`]. +#[cfg(feature = "mysql")] +pub const MYSQL_JOB_COLUMNS: &str = crate::database::mysql::JOB_COLUMNS; /// Columns River selects to decode a SQLite job row, for use with /// [`decode_sqlite_job_row`]. #[cfg(feature = "sqlite")] pub const SQLITE_JOB_COLUMNS: &str = crate::database::sqlite::JOB_COLUMNS; +/// Decodes a row selected with [`SQLITE_JOB_COLUMNS`] exactly as River +/// decodes its own rows. +/// +/// # Errors +/// +/// Returns an error when the row can't be decoded. +#[cfg(feature = "mysql")] +pub fn decode_mysql_job_row(row: &sqlx::mysql::MySqlRow) -> Result { + crate::database::mysql::decode_job_row(row).map_err(|undecodable| undecodable.error.into()) +} /// Decodes a row selected with [`SQLITE_JOB_COLUMNS`] exactly as River /// decodes its own rows. /// @@ -1193,6 +1280,13 @@ pub fn decode_sqlite_job_row(row: &sqlx::sqlite::SqliteRow) -> Result ClaimedJob { + ClaimedJob::from_decoded(crate::database::mysql::decode_job_row(row)) +} /// Decodes a claimed row selected with [`SQLITE_JOB_COLUMNS`] as far as /// River can, for [`PilotProducer::claim`]. #[cfg(feature = "sqlite")] diff --git a/rust/riverqueue/src/client/completer.rs b/rust/riverqueue/src/client/completer.rs index 5bb3ec2e2..f5c0fe459 100644 --- a/rust/riverqueue/src/client/completer.rs +++ b/rust/riverqueue/src/client/completer.rs @@ -28,6 +28,8 @@ use tracing::{debug, error}; use crate::__private::{DatabaseConnection as PilotDatabaseConnection, JobSetStateParams}; use crate::client::attempts::remove_running_attempt; use crate::client::backoff::exponential_backoff; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::client::{ClientInner, RuntimeNotification, peers, tolerant_row}; @@ -144,6 +146,8 @@ impl CompletionBatcher { let backend_concurrency = match inner.database.kind() { #[cfg(feature = "postgres")] DatabaseKind::Postgres => COMPLETION_POSTGRES_CONCURRENCY, + #[cfg(feature = "mysql")] + DatabaseKind::MySql => 1, #[cfg(feature = "sqlite")] DatabaseKind::Sqlite => 1, }; @@ -485,8 +489,8 @@ async fn persist_with_retries( } /// Wakes producers for jobs that client shutdown returned to `available`, so -/// another client picks them up without waiting for its next poll. SQLite -/// writes its durable wakeup inside the completion transaction instead. +/// another client picks them up without waiting for its next poll. MySQL and +/// SQLite write their durable wakeups inside the completion transaction. #[cfg_attr( not(feature = "postgres"), allow(clippy::unused_async, reason = "only Postgres sends a notification") @@ -511,7 +515,9 @@ async fn notify_interrupted_jobs(inner: &ClientInner, batch: &[CompletionUpdate] .send(RuntimeNotification::Insert((*queue).to_owned())); } match inner.database.pool() { - // SQLite clients wrote these notifications with the completion. + // Outbox clients wrote these notifications with the completion. + #[cfg(feature = "mysql")] + DatabasePool::MySql(_) => {} #[cfg(feature = "sqlite")] DatabasePool::Sqlite(_) => {} #[cfg(feature = "postgres")] @@ -558,6 +564,73 @@ pub(super) async fn persist_completion_batch( batch: &[CompletionUpdate], ) -> Result, Error> { match inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let mut rows = Vec::with_capacity(batch.len()); + let now = Utc::now(); + for update in batch { + let row = crate::database::mysql::complete_decoded( + &mut transaction, + &crate::database::mysql::CompleteJob { + attempt: update.attempt, + error: update.error.as_ref(), + finalized_at: update.finalized_at, + id: update.job_id, + metadata_updates: (!update.metadata.is_empty()).then_some(&update.metadata), + now, + scheduled_at: update.scheduled_at, + state: update.state, + }, + ) + .await + .map_err(mysql_backend_error)?; + let row = match row { + Some(row) => Some(row), + None => crate::database::mysql::merge_metadata_if_not_running( + &mut transaction, + update.job_id, + &update.metadata, + ) + .await + .map_err(mysql_backend_error)?, + }; + rows.extend(row.and_then(tolerant_row)); + } + let interrupted_queues = rows + .iter() + .filter(|row| row.state == JobState::Available) + .filter(|row| { + batch.iter().any(|update| { + update.job_id == row.id && update.event_kind == JobEventKind::Interrupted + }) + }) + .map(|row| row.queue.as_str()) + .collect::>(); + for queue in interrupted_queues { + let payload = serde_json::json!({ "queue": queue }).to_string(); + crate::database::mysql::notification_insert( + &mut transaction, + &[crate::database::mysql::NotificationInput { + payload: &payload, + topic: crate::NOTIFICATION_TOPIC_INSERT, + }], + ) + .await + .map_err(mysql_backend_error)?; + } + if inner.pilot.intercepts_job_set_state() { + after_jobs_set_state( + inner, + PilotDatabaseConnection::MySql(&mut transaction), + &batch.iter().map(|update| update.job_id).collect::>(), + &rows, + ) + .await?; + } + transaction.commit().await?; + Ok(rows) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; diff --git a/rust/riverqueue/src/client/insert.rs b/rust/riverqueue/src/client/insert.rs index 81c136fb5..f837be294 100644 --- a/rust/riverqueue/src/client/insert.rs +++ b/rust/riverqueue/src/client/insert.rs @@ -33,6 +33,8 @@ use crate::__private::{ DatabaseConnection as PilotDatabaseConnection, JobInsertParams as PilotJobInsertParams, }; use crate::client::RuntimeNotification; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; use crate::client::request::Target; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; @@ -594,6 +596,19 @@ impl Client { transaction.commit().await?; inserted } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let inserted = self + .insert_and_finish( + PilotDatabaseConnection::MySql(&mut transaction), + jobs, + finish, + ) + .await?; + transaction.commit().await?; + inserted + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -807,6 +822,23 @@ impl Client { .execute(connection) .await?; } + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => { + let payloads = queues + .into_iter() + .map(crate::protocol::insert_notification_payload) + .collect::>(); + let notifications = payloads + .iter() + .map(|payload| crate::database::mysql::NotificationInput { + payload, + topic: crate::protocol::NOTIFICATION_TOPIC_INSERT, + }) + .collect::>(); + crate::database::mysql::notification_insert(connection, ¬ifications) + .await + .map_err(mysql_backend_error)?; + } #[cfg(feature = "sqlite")] PilotDatabaseConnection::Sqlite(connection) => { let payloads = queues @@ -834,18 +866,19 @@ impl Client { connection: PilotDatabaseConnection<'_>, job: InsertContext, ) -> Result { - let InsertContext { - encoded_args, - kind, - opts, - state, - created_at, - unique_key, - unique_states, - } = job; match connection { #[cfg(feature = "postgres")] PilotDatabaseConnection::Postgres(connection) => { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + let table = self.inner.schema.qualify("river_job"); let state_type = self.inner.schema.qualify("river_job_state"); let state_function = self.inner.schema.qualify("river_job_state_in_bitmask"); @@ -897,46 +930,112 @@ impl Client { let duplicate = nonce.map_or(duplicate, |nonce| lacks_nonce(&row, &nonce)); Ok(InsertedJob::new(row, duplicate)) } + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => insert_mysql_row(connection, job).await, #[cfg(feature = "sqlite")] - PilotDatabaseConnection::Sqlite(connection) => { - // Like Go's SQLite driver, every inserted row carries a nonce, - // and times left unset are filled in by SQLite's own clock. - let nonce = unique_insert_nonce(); - let inserted = crate::database::sqlite::insert( - connection, - &crate::database::sqlite::InsertJob { - attempt: 0, - attempted_at: None, - attempted_by: &[], - created_at, - encoded_args: &encoded_args, - errors: &[], - finalized_at: None, - id: None, - kind: &kind, - max_attempts: opts.max_attempts, - metadata: &opts.metadata, - priority: opts.priority, - queue: &opts.queue, - scheduled_at: opts.scheduled_at, - state, - tags: &opts.tags, - unique_key: unique_key.as_deref(), - unique_nonce: Some(&nonce), - unique_states, - }, - ) - .await - .map_err(sqlite_backend_error)?; - Ok(InsertedJob::new( - inserted.job, - inserted.unique_skipped_as_duplicate, - )) - } + PilotDatabaseConnection::Sqlite(connection) => insert_sqlite_row(connection, job).await, } } } +#[cfg(feature = "mysql")] +async fn insert_mysql_row( + connection: &mut sqlx::MySqlConnection, + job: InsertContext, +) -> Result { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + // Like Go's MySQL driver, every inserted row carries a nonce, + // and times left unset are filled in by MySQL's own clock. + let nonce = unique_insert_nonce(); + let inserted = crate::database::mysql::insert( + connection, + &crate::database::mysql::InsertJob { + attempt: 0, + attempted_at: None, + attempted_by: &[], + created_at, + encoded_args: &encoded_args, + errors: &[], + finalized_at: None, + id: None, + kind: &kind, + max_attempts: opts.max_attempts, + metadata: &opts.metadata, + priority: opts.priority, + queue: &opts.queue, + scheduled_at: opts.scheduled_at, + state, + tags: &opts.tags, + unique_key: unique_key.as_deref(), + unique_nonce: Some(&nonce), + unique_states, + }, + ) + .await + .map_err(mysql_backend_error)?; + Ok(InsertedJob::new( + inserted.job, + inserted.unique_skipped_as_duplicate, + )) +} + +#[cfg(feature = "sqlite")] +async fn insert_sqlite_row( + connection: &mut sqlx::SqliteConnection, + job: InsertContext, +) -> Result { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + // Like Go's SQLite driver, every inserted row carries a nonce, + // and times left unset are filled in by SQLite's own clock. + let nonce = unique_insert_nonce(); + let inserted = crate::database::sqlite::insert( + connection, + &crate::database::sqlite::InsertJob { + attempt: 0, + attempted_at: None, + attempted_by: &[], + created_at, + encoded_args: &encoded_args, + errors: &[], + finalized_at: None, + id: None, + kind: &kind, + max_attempts: opts.max_attempts, + metadata: &opts.metadata, + priority: opts.priority, + queue: &opts.queue, + scheduled_at: opts.scheduled_at, + state, + tags: &opts.tags, + unique_key: unique_key.as_deref(), + unique_nonce: Some(&nonce), + unique_states, + }, + ) + .await + .map_err(sqlite_backend_error)?; + Ok(InsertedJob::new( + inserted.job, + inserted.unique_skipped_as_duplicate, + )) +} + /// Exposes the mutable fields of each job to an extension's insert hook. fn extension_insert_params(jobs: &mut [InsertContext]) -> Vec> { jobs.iter_mut() diff --git a/rust/riverqueue/src/client/jobs.rs b/rust/riverqueue/src/client/jobs.rs index ac8482870..21aef3c9c 100644 --- a/rust/riverqueue/src/client/jobs.rs +++ b/rust/riverqueue/src/client/jobs.rs @@ -314,7 +314,9 @@ impl JobGetRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session.storage(inner).job_get(self.id).await + let result = session.storage(inner).job_get(self.id).await; + session.commit().await?; + result } } @@ -329,6 +331,7 @@ impl JobListRequest<'_> { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; let jobs = session.storage(inner).job_list(&self.params).await?; + session.commit().await?; let last_cursor = jobs .last() .map(|job| JobListCursor::after_job(job, &self.params)); @@ -361,11 +364,13 @@ request_type! { impl JobUpdateRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; - let mut session = self.target.session(inner, Access::Autocommit).await?; - session + let mut session = self.target.session(inner, Access::Transaction).await?; + let row = session .storage(inner) .job_update(self.id, self.params) - .await + .await?; + session.commit().await?; + Ok(row) } } diff --git a/rust/riverqueue/src/client/mod.rs b/rust/riverqueue/src/client/mod.rs index 68b517853..9cb910093 100644 --- a/rust/riverqueue/src/client/mod.rs +++ b/rust/riverqueue/src/client/mod.rs @@ -36,7 +36,7 @@ pub use self::queues::{ QueueGetRequest, QueueListRequest, QueuePauseRequest, QueueResumeRequest, QueueUpdateRequest, Queues, }; -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] pub(crate) use self::record::FieldErrors; pub(crate) use self::record::{DecodedJob, UndecodableJob, tolerant_row}; #[cfg(feature = "postgres")] @@ -157,6 +157,10 @@ pub(crate) struct ClientInner { pub(crate) workers: Workers, } +#[cfg(feature = "mysql")] +fn mysql_backend_error(error: crate::database::mysql::BackendError) -> Error { + Error::Database(error.into()) +} #[cfg(feature = "sqlite")] fn sqlite_backend_error(error: crate::database::sqlite::BackendError) -> Error { Error::Database(error.into()) @@ -197,7 +201,7 @@ impl ClientInner { } /// Whether this client hears committed notifications through a notifier, - /// a Postgres listener or SQLite outbox poller, like River Go's client + /// a Postgres listener or MySQL/SQLite outbox poller, like River Go's client /// notifier. A poll-only client has none, and neither does a client of a /// Postgres server without `LISTEN`/`NOTIFY` once that's detected. pub(crate) fn has_notifier(&self) -> bool { @@ -219,6 +223,8 @@ impl ClientInner { #[cfg(feature = "postgres")] schema: self.schema.clone(), }, + #[cfg(feature = "mysql")] + DatabasePool::MySql(_) => PilotDatabaseConfig::MySql, #[cfg(feature = "sqlite")] DatabasePool::Sqlite(_) => PilotDatabaseConfig::Sqlite, } @@ -236,6 +242,8 @@ impl ClientInner { match self.database.pool() { #[cfg(feature = "postgres")] DatabasePool::Postgres(pool) => PilotDatabasePool::Postgres(pool.clone()), + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => PilotDatabasePool::MySql(pool.clone()), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => PilotDatabasePool::Sqlite(pool.clone()), } diff --git a/rust/riverqueue/src/client/notifier.rs b/rust/riverqueue/src/client/notifier.rs index be9aa976e..44173e9ba 100644 --- a/rust/riverqueue/src/client/notifier.rs +++ b/rust/riverqueue/src/client/notifier.rs @@ -24,6 +24,8 @@ use crate::Error; use crate::client::ClientInner; use crate::client::attempts::signal_running_attempt; use crate::client::backoff::exponential_backoff; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::maintenance::LeadershipWakeup; @@ -350,6 +352,103 @@ async fn listen_until_error( } } +/// Polls MySQL's notification outbox until cancelled. +/// +/// Like River Go's MySQL listener, the first successful poll connects after +/// the outbox's current maximum ID and subscribes to River's topics, so +/// earlier rows are never delivered. Poll failures are logged and retried +/// with River's service backoff. The cursor is kept, so no durable +/// notification is skipped. +#[cfg(feature = "mysql")] +pub(super) async fn run_mysql_notifications( + inner: Arc, + pool: sqlx::MySqlPool, + cancel: CancellationToken, + queue_notifications: broadcast::Sender, + ready: ReadySlot, +) -> Result<(), Error> { + let mut attempt = 0; + let mut listener = crate::database::mysql::NotificationListener::default(); + let mut notification_tick = + tokio::time::interval(crate::database::mysql::DEFAULT_NOTIFICATION_POLL_INTERVAL); + notification_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + () = cancel.cancelled() => return Ok(()), + _ = notification_tick.tick() => {} + } + // Like the rest of a poll, waiting for a connection during an + // outage must not delay a stop, as Go's cancellable context doesn't. + let connection = tokio::select! { + biased; + () = cancel.cancelled() => return Ok(()), + connection = pool.acquire() => connection, + }; + let polled = async { + let mut connection = connection?; + let receive = async { + if !listener.is_connected() { + listener + .connect(&mut connection) + .await + .map_err(mysql_backend_error)?; + } + for topic in [ + crate::NOTIFICATION_TOPIC_CONTROL, + crate::NOTIFICATION_TOPIC_INSERT, + crate::NOTIFICATION_TOPIC_LEADERSHIP, + ] { + listener + .listen(&mut connection, topic) + .await + .map_err(mysql_backend_error)?; + } + report_ready(&ready); + while let Some(notification) = listener + .next(&mut connection) + .await + .map_err(mysql_backend_error)? + { + dispatch_notification( + &inner, + &queue_notifications, + ¬ification.topic, + ¬ification.payload, + ); + } + Ok::<_, Error>(()) + }; + tokio::select! { + biased; + () = cancel.cancelled() => { + // Discard a connection with an outstanding locking read so + // neither shutdown nor the next borrower waits for its result. + drop(connection.detach()); + Ok(()) + } + result = receive => result, + } + } + .await; + if let Err(poll_error) = polled { + attempt += 1; + let sleep = exponential_backoff(attempt); + error!( + attempt, + error = %crate::error::Chain(&poll_error), + sleep_duration = ?sleep, + "River notification poll failed (will retry after backoff); producers keep polling" + ); + tokio::select! { + () = cancel.cancelled() => return Ok(()), + () = tokio::time::sleep(sleep) => {} + } + continue; + } + attempt = 0; + } +} /// Polls SQLite's notification outbox until cancelled. /// /// Like River Go's SQLite listener, the first successful poll connects after diff --git a/rust/riverqueue/src/client/producer.rs b/rust/riverqueue/src/client/producer.rs index fa68924f9..7f03c6664 100644 --- a/rust/riverqueue/src/client/producer.rs +++ b/rust/riverqueue/src/client/producer.rs @@ -22,6 +22,8 @@ use crate::client::attempts::{ use crate::client::backoff::{RestartBackoff, exponential_backoff}; use crate::client::completer::CompletionUpdate; use crate::client::executor::{AbortOnDrop, execute_job}; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::client::{ @@ -1032,6 +1034,8 @@ async fn fetch_available( let use_parallel_fetch = match inner.database.kind() { #[cfg(feature = "postgres")] DatabaseKind::Postgres => true, + #[cfg(feature = "mysql")] + DatabaseKind::MySql => true, #[cfg(feature = "sqlite")] DatabaseKind::Sqlite => false, }; @@ -1130,6 +1134,20 @@ pub(crate) async fn standard_claim( .iter() .map(decode_job_row) .collect()), + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => { + let params = crate::database::mysql::ClaimJobs { + client_id: &inner.id, + kinds: inner.fetch_kinds.as_deref(), + limit, + max_attempted_by: ATTEMPTED_BY_MAX, + now: Utc::now(), + queue, + }; + crate::database::mysql::claim(connection, ¶ms) + .await + .map_err(mysql_backend_error) + } #[cfg(feature = "sqlite")] PilotDatabaseConnection::Sqlite(connection) => { let params = crate::database::sqlite::ClaimJobs { @@ -1195,6 +1213,24 @@ pub(super) async fn fetch_jobs( ) .await? } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let Some(transaction) = + unless_cancelled(cancel, crate::database::mysql::Transaction::begin(pool)).await + else { + return Ok(FetchedJobs::default()); + }; + let mut transaction = transaction?; + let rows = standard_claim( + inner, + PilotDatabaseConnection::MySql(&mut transaction), + queue, + maximum, + ) + .await?; + transaction.commit().await?; + rows + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let Some(connection) = unless_cancelled(cancel, pool.acquire()).await else { diff --git a/rust/riverqueue/src/client/queues.rs b/rust/riverqueue/src/client/queues.rs index 16dc4e217..84b9e104c 100644 --- a/rust/riverqueue/src/client/queues.rs +++ b/rust/riverqueue/src/client/queues.rs @@ -163,11 +163,9 @@ impl QueueGetRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session - .storage(inner) - .queue_get(&self.name) - .await? - .ok_or_else(|| Error::NotFound(crate::Record::Queue(self.name.clone()))) + let queue = session.storage(inner).queue_get(&self.name).await?; + session.commit().await?; + queue.ok_or_else(|| Error::NotFound(crate::Record::Queue(self.name.clone()))) } } @@ -181,7 +179,9 @@ impl QueueListRequest<'_> { async fn run(self) -> Result, Error> { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session.storage(inner).queue_list(&self.params).await + let queues = session.storage(inner).queue_list(&self.params).await?; + session.commit().await?; + Ok(queues) } } diff --git a/rust/riverqueue/src/client/run.rs b/rust/riverqueue/src/client/run.rs index f8c42bac0..db3f264f0 100644 --- a/rust/riverqueue/src/client/run.rs +++ b/rust/riverqueue/src/client/run.rs @@ -13,6 +13,8 @@ use tracing::{debug, error, warn}; use crate::client::ClientInner; use crate::client::backoff::RestartBackoff; use crate::client::completer::run_completion_batcher; +#[cfg(feature = "mysql")] +use crate::client::notifier::run_mysql_notifications; #[cfg(feature = "postgres")] use crate::client::notifier::run_notifications; #[cfg(feature = "sqlite")] @@ -28,8 +30,8 @@ type ShutdownSignal = std::pin::Pin + S impl Client { /// Starts configured queues and returns a lifecycle handle. /// - /// The client supervises its services: a notification listener, SQLite - /// outbox poller, maintenance, or extension service that fails is logged + /// The client supervises its services: a notification listener, MySQL or + /// SQLite outbox poller, maintenance, or extension service that fails is logged /// and restarted with backoff, and producers keep polling meanwhile, so a /// database outage never stops the client. Only a failure of the /// producers or the completer, which would leave jobs unworked or @@ -399,6 +401,14 @@ impl Supervisor { inner.queue_notifications.clone(), ready, )), + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => Box::pin(run_mysql_notifications( + Arc::clone(&inner), + pool.clone(), + cancel.clone(), + inner.queue_notifications.clone(), + ready, + )), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => Box::pin(run_sqlite_notifications( Arc::clone(&inner), diff --git a/rust/riverqueue/src/client/tests.rs b/rust/riverqueue/src/client/tests.rs index d52f83f76..a3483a2d5 100644 --- a/rust/riverqueue/src/client/tests.rs +++ b/rust/riverqueue/src/client/tests.rs @@ -151,7 +151,9 @@ async fn go_notification_fixtures_dispatch_correctly() { // in both Postgres-only and SQLite-only builds. #[cfg(feature = "postgres")] let pool = sqlx::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); - #[cfg(not(feature = "postgres"))] + #[cfg(all(not(feature = "postgres"), feature = "mysql"))] + let pool = sqlx::MySqlPool::connect_lazy("mysql://localhost/unused").unwrap(); + #[cfg(all(not(feature = "postgres"), not(feature = "mysql"), feature = "sqlite"))] let pool = sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(); for name in names { diff --git a/rust/riverqueue/src/database.rs b/rust/riverqueue/src/database.rs index 8dc343350..1e848ee74 100644 --- a/rust/riverqueue/src/database.rs +++ b/rust/riverqueue/src/database.rs @@ -11,6 +11,8 @@ use std::time::Duration; #[cfg(feature = "postgres")] use chrono::NaiveTime; pub use riverqueue_migrate::{SchemaName, SchemaNameError}; +#[cfg(feature = "mysql")] +use sqlx::{MySql, MySqlConnection, MySqlPool}; #[cfg(feature = "postgres")] use sqlx::{PgConnection, PgPool, Postgres}; #[cfg(feature = "sqlite")] @@ -58,6 +60,29 @@ pub async fn begin_postgres(pool: &PgPool) -> Result Result, sqlx::Error> { + let pool = pool.clone(); + run_to_completion(async move { + pool.begin_with("SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN") + .await + }) + .await +} /// Begins a SQLite transaction that may write, protected like /// `begin_postgres` against being abandoned half-started. /// @@ -111,6 +136,8 @@ async fn run_to_completion( } pub(crate) mod erased; +#[cfg(feature = "mysql")] +pub(crate) mod mysql; #[cfg(feature = "postgres")] pub(crate) mod postgres_capabilities; #[cfg(feature = "sqlite")] @@ -130,6 +157,9 @@ pub enum ClientDatabase<'a> { /// A Postgres client's pool and options. #[cfg(feature = "postgres")] Postgres(&'a PostgresDatabase), + /// A MySQL client's pool. + #[cfg(feature = "mysql")] + MySql(&'a MySqlDatabase), /// A SQLite client's pool. #[cfg(feature = "sqlite")] Sqlite(&'a SqliteDatabase), @@ -143,11 +173,25 @@ impl<'a> ClientDatabase<'a> { pub const fn as_postgres(&self) -> Option<&'a PostgresDatabase> { match self { Self::Postgres(database) => Some(database), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } } + /// Returns the MySQL pool, or `None` for another backend. + #[cfg(feature = "mysql")] + #[must_use] + pub const fn as_mysql(&self) -> Option<&'a MySqlDatabase> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(database) => Some(database), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } /// Returns the SQLite pool, or `None` for another backend. #[cfg(feature = "sqlite")] #[must_use] @@ -156,6 +200,8 @@ impl<'a> ClientDatabase<'a> { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(database) => Some(database), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } @@ -165,6 +211,8 @@ impl<'a> ClientDatabase<'a> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -181,6 +229,10 @@ pub enum DatabaseKind { /// SQLite and compatible implementations. #[cfg(feature = "sqlite")] Sqlite, + // Append backends to preserve existing discriminants. + /// MySQL and compatible implementations. + #[cfg(feature = "mysql")] + MySql, } impl fmt::Display for DatabaseKind { @@ -188,6 +240,8 @@ impl fmt::Display for DatabaseKind { formatter.write_str(match self { #[cfg(feature = "postgres")] Self::Postgres => "Postgres", + #[cfg(feature = "mysql")] + Self::MySql => "MySQL", #[cfg(feature = "sqlite")] Self::Sqlite => "SQLite", }) @@ -413,6 +467,12 @@ impl Default for PostgresReindexSchedule { } } +/// A MySQL source and its backend-specific River options. +#[cfg(feature = "mysql")] +#[derive(Clone)] +pub struct MySqlDatabase { + pool: MySqlPool, +} /// A SQLite source and its backend-specific River options. #[cfg(feature = "sqlite")] #[derive(Clone)] @@ -420,6 +480,20 @@ pub struct SqliteDatabase { pool: SqlitePool, } +#[cfg(feature = "mysql")] +impl MySqlDatabase { + /// Uses a MySQL pool. + #[must_use] + pub const fn new(pool: MySqlPool) -> Self { + Self { pool } + } + + /// Returns the underlying SQLx pool. + #[must_use] + pub const fn pool(&self) -> &MySqlPool { + &self.pool + } +} #[cfg(feature = "sqlite")] impl SqliteDatabase { /// Uses a SQLite pool. @@ -435,6 +509,14 @@ impl SqliteDatabase { } } +#[cfg(feature = "mysql")] +impl fmt::Debug for MySqlDatabase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("MySqlDatabase") + .finish_non_exhaustive() + } +} #[cfg(feature = "sqlite")] impl fmt::Debug for SqliteDatabase { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -483,12 +565,20 @@ impl IntoDatabase for &PgPool {} impl IntoDatabase for PostgresDatabase {} #[cfg(feature = "postgres")] impl IntoDatabase for &PostgresDatabase {} +#[cfg(feature = "mysql")] +impl IntoDatabase for MySqlPool {} #[cfg(feature = "sqlite")] impl IntoDatabase for SqlitePool {} +#[cfg(feature = "mysql")] +impl IntoDatabase for &MySqlPool {} #[cfg(feature = "sqlite")] impl IntoDatabase for &SqlitePool {} +#[cfg(feature = "mysql")] +impl IntoDatabase for MySqlDatabase {} #[cfg(feature = "sqlite")] impl IntoDatabase for SqliteDatabase {} +#[cfg(feature = "mysql")] +impl IntoDatabase for &MySqlDatabase {} #[cfg(feature = "sqlite")] impl IntoDatabase for &SqliteDatabase {} @@ -551,6 +641,11 @@ impl<'executor> DatabaseTransactionExecutor<'executor> for &'executor mut sqlx::Transaction<'_, sqlx::Postgres> { } +#[cfg(feature = "mysql")] +impl<'executor> DatabaseTransactionExecutor<'executor> + for &'executor mut sqlx::Transaction<'_, sqlx::MySql> +{ +} #[cfg(feature = "sqlite")] impl<'executor> DatabaseTransactionExecutor<'executor> for &'executor mut sqlx::Transaction<'_, sqlx::Sqlite> @@ -573,6 +668,8 @@ pub(crate) fn into_database(database: D) -> Database { pub(crate) enum DatabasePool<'pool> { #[cfg(feature = "postgres")] Postgres(&'pool PgPool), + #[cfg(feature = "mysql")] + MySql(&'pool MySqlPool), #[cfg(feature = "sqlite")] Sqlite(&'pool SqlitePool), } @@ -582,6 +679,8 @@ mod private { Database, DatabaseConnection, DatabaseInner, DatabaseKind, ErasedExecutor, ErasedTransaction, ExecutorInner, PoolConnection, Transaction, }; + #[cfg(feature = "mysql")] + use super::{MySql, MySqlConnection, MySqlDatabase, MySqlPool}; #[cfg(feature = "postgres")] use super::{PgConnection, PgPool, Postgres, PostgresDatabase}; #[cfg(feature = "sqlite")] @@ -652,6 +751,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for MySqlPool { + fn erase(self) -> Database { + MySqlDatabase::new(self).erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for SqlitePool { fn erase(self) -> Database { @@ -659,6 +764,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for &MySqlPool { + fn erase(self) -> Database { + self.clone().erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for &SqlitePool { fn erase(self) -> Database { @@ -666,6 +777,14 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for MySqlDatabase { + fn erase(self) -> Database { + Database { + inner: DatabaseInner::MySql(self), + } + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for SqliteDatabase { fn erase(self) -> Database { @@ -675,6 +794,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for &MySqlDatabase { + fn erase(self) -> Database { + self.clone().erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for &SqliteDatabase { fn erase(self) -> Database { @@ -719,6 +844,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut MySqlConnection { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseConnection::MySql(self)) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut SqliteConnection { fn erase(self) -> ErasedExecutor<'executor> { @@ -726,6 +857,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut PoolConnection { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseConnection::MySql(self.as_mut())) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut PoolConnection { fn erase(self) -> ErasedExecutor<'executor> { @@ -733,6 +870,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor MySqlPool { + fn erase(self) -> ErasedExecutor<'executor> { + pool_executor(DatabaseKind::MySql) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor SqlitePool { fn erase(self) -> ErasedExecutor<'executor> { @@ -740,6 +883,22 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut Transaction<'_, MySql> { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseTransactionExecutorSealed::connection(self)) + } + } + + #[cfg(feature = "mysql")] + impl<'executor> DatabaseTransactionExecutorSealed<'executor> + for &'executor mut Transaction<'_, MySql> + { + fn connection(self) -> DatabaseConnection<'executor> { + DatabaseConnection::MySql(self.as_mut()) + } + } + #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut Transaction<'_, Sqlite> { fn erase(self) -> ErasedExecutor<'executor> { diff --git a/rust/riverqueue/src/database/erased.rs b/rust/riverqueue/src/database/erased.rs index a4e9b93c2..e909dd94c 100644 --- a/rust/riverqueue/src/database/erased.rs +++ b/rust/riverqueue/src/database/erased.rs @@ -4,6 +4,8 @@ use std::fmt; +#[cfg(feature = "mysql")] +use super::MySqlDatabase; #[cfg(feature = "sqlite")] use super::SqliteDatabase; use crate::__private::DatabaseConnection; @@ -40,6 +42,8 @@ impl Database { ) { match &mut self.inner { DatabaseInner::Postgres(source) => source.extend_default_reindex_names(names), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => {} #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => {} } @@ -50,6 +54,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => ClientDatabase::Postgres(source), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(source) => ClientDatabase::MySql(source), #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(source) => ClientDatabase::Sqlite(source), } @@ -61,6 +67,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => DatabaseKind::Sqlite, } @@ -76,6 +84,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => source.capabilities().supports_listen_notify(), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => false, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => false, } @@ -88,6 +98,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => source.capabilities().supports_listen_notify(), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => true, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => true, } @@ -97,7 +109,7 @@ impl Database { /// another backend. #[cfg(feature = "postgres")] #[cfg_attr( - not(feature = "sqlite"), + not(any(feature = "sqlite", feature = "mysql")), expect( clippy::unnecessary_wraps, reason = "another backend may be compiled in" @@ -108,6 +120,8 @@ impl Database { ) -> Option<&super::postgres_capabilities::CapabilitiesCache> { match &self.inner { DatabaseInner::Postgres(source) => Some(source.capabilities()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -120,6 +134,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => Some(source.schema()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -127,7 +143,7 @@ impl Database { #[cfg(feature = "postgres")] #[cfg_attr( - not(feature = "sqlite"), + not(any(feature = "sqlite", feature = "mysql")), expect( clippy::unnecessary_wraps, reason = "another backend may be compiled in" @@ -136,6 +152,8 @@ impl Database { pub(crate) fn postgres_reindex(&self) -> Option<&PostgresReindexConfig> { match &self.inner { DatabaseInner::Postgres(source) => Some(source.reindex()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -206,6 +224,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => DatabasePool::Postgres(source.pool()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(source) => DatabasePool::MySql(source.pool()), #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(source) => DatabasePool::Sqlite(source.pool()), } @@ -226,6 +246,8 @@ impl fmt::Debug for Database { pub(crate) enum DatabaseInner { #[cfg(feature = "postgres")] Postgres(PostgresDatabase), + #[cfg(feature = "mysql")] + MySql(MySqlDatabase), #[cfg(feature = "sqlite")] Sqlite(SqliteDatabase), } diff --git a/rust/riverqueue/src/database/mysql.rs b/rust/riverqueue/src/database/mysql.rs new file mode 100644 index 000000000..317a48543 --- /dev/null +++ b/rust/riverqueue/src/database/mysql.rs @@ -0,0 +1,1550 @@ +//! MySQL operation primitives. +//! +//! MySQL deliberately stays behind River's sealed database boundary. These +//! operations mirror River's storage semantics without exposing SQLx executor +//! types through the public API. Multi-step operations are expressed as small +//! primitives so their caller can compose them in one MySQL transaction. + +#![allow( + clippy::needless_raw_string_hashes, + reason = "consistent SQL delimiters make large dialect-specific statements easier to audit" +)] +#![allow( + clippy::struct_field_names, + reason = "leader_id is the cross-language River protocol field name" +)] + +use std::{ + collections::{HashMap, VecDeque}, + time::Duration, +}; + +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value, value::RawValue}; +use sqlx::{AssertSqlSafe, Connection as _, FromRow, MySql, MySqlConnection, QueryBuilder, Row}; + +use sqlx::mysql::MySqlRow; + +use crate::{ + AttemptError, JobMetadata, JobRow, JobState, METADATA_KEY_UNIQUE_NONCE, Queue, + client::{DecodedJob, FieldErrors, UndecodableJob, go_time_json, tolerant_row}, + query::{JobListKeyset, JobListSqlPart}, +}; + +mod connection; + +pub(crate) use connection::{Connection, Transaction}; + +/// MySQL JSON values are projected as bytes so decoding preserves large integers. +pub(crate) const JOB_COLUMNS: &str = r#" + id, attempt, attempted_at, CAST(attempted_by AS BINARY) AS attempted_by, + created_at, CAST(args AS BINARY) AS encoded_args, CAST(errors AS BINARY) AS errors, + finalized_at, kind, max_attempts, CAST(metadata AS BINARY) AS metadata, + priority, queue, scheduled_at, state, CAST(COALESCE(tags, 'null') AS BINARY) AS tags, + unique_key, unique_states +"#; +const QUEUE_COLUMNS: &str = + "created_at, CAST(metadata AS CHAR) AS metadata, name, paused_at, updated_at"; +const ERRORS_APPENDED: &str = "JSON_ARRAY_APPEND(CASE WHEN errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(errors) = 'ARRAY' THEN errors ELSE JSON_ARRAY(errors) END, '$', CAST(? AS JSON))"; + +/// A short poll interval keeps local wakeups responsive while queue fetch +/// polling remains the durable recovery path. +pub(crate) const DEFAULT_NOTIFICATION_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Maximum number of outbox rows a notification listener reads at once, +/// matching River Go's MySQL listener. +pub(crate) const NOTIFICATION_BATCH_SIZE: i64 = 256; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum BackendError { + #[error("invalid MySQL River row: {0}")] + InvalidRow(String), + #[error(transparent)] + Sqlx(#[from] sqlx::Error), + #[error("invalid MySQL River JSON")] + Json(#[from] serde_json::Error), +} + +/// Reports MySQL failures the way Postgres's reach callers: as SQLx's own +/// error, with stored values River can't decode as [`sqlx::Error::Decode`]. +impl From for sqlx::Error { + fn from(error: BackendError) -> Self { + match error { + BackendError::Sqlx(error) => error, + BackendError::InvalidRow(message) => { + Self::Decode(format!("invalid MySQL River row: {message}").into()) + } + BackendError::Json(error) => Self::Decode(Box::new(error)), + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct InsertJob<'a> { + pub attempted_at: Option>, + pub attempted_by: &'a [String], + pub attempt: i32, + /// When unset, MySQL's `UTC_TIMESTAMP(6)`, like Go's driver. + pub created_at: Option>, + pub encoded_args: &'a serde_json::value::RawValue, + pub errors: &'a [AttemptError], + pub finalized_at: Option>, + pub id: Option, + pub kind: &'a str, + pub max_attempts: i32, + pub metadata: &'a JobMetadata, + pub priority: i16, + pub queue: &'a str, + /// When unset, MySQL's `UTC_TIMESTAMP(6)`, like Go's driver. + pub scheduled_at: Option>, + pub state: JobState, + pub tags: &'a [String], + pub unique_key: Option<&'a [u8]>, + pub unique_nonce: Option<&'a str>, + pub unique_states: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct InsertedJob { + pub job: JobRow, + pub unique_skipped_as_duplicate: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct ClaimJobs<'a> { + pub client_id: &'a str, + /// Kinds to claim, or `None` for every kind. Ignored when claiming + /// selected IDs. + pub kinds: Option<&'a [String]>, + pub limit: i32, + pub max_attempted_by: i32, + pub now: DateTime, + pub queue: &'a str, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct ListJobs<'a> { + /// Excludes running jobs before applying the limit, as bulk deletion does. + pub exclude_running: bool, + pub ids: &'a [i64], + pub keyset: JobListKeyset, + pub kinds: &'a [&'a str], + pub limit: i32, + pub metadata: Option<&'a Map>, + pub priorities: &'a [i16], + pub queues: &'a [&'a str], + pub states: &'a [JobState], + pub tags_all: &'a [&'a str], + pub tags_any: &'a [&'a str], +} + +#[derive(Clone, Debug)] +pub(crate) struct CompleteJob<'a> { + pub attempt: Option, + pub error: Option<&'a AttemptError>, + pub finalized_at: Option>, + pub id: i64, + pub metadata_updates: Option<&'a Map>, + pub now: DateTime, + pub scheduled_at: Option>, + pub state: JobState, +} + +#[derive(Clone, Debug)] +pub(crate) struct RescueJob<'a> { + pub error: &'a AttemptError, + pub finalized_at: Option>, + pub id: i64, + pub scheduled_at: DateTime, + pub state: JobState, + /// The rescue applies only to a job still running from before this time. + pub stuck_horizon: DateTime, +} + +/// Job cleaner deletion horizons; `None` keeps that state indefinitely. +#[derive(Clone, Debug)] +pub(crate) struct CleanupJobs<'a> { + pub cancelled_before: Option>, + pub completed_before: Option>, + pub discarded_before: Option>, + pub limit: i32, + pub queues_excluded: &'a [&'a str], + pub queues_included: Option<&'a [&'a str]>, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Leader { + pub elected_at: DateTime, + pub expires_at: DateTime, + pub leader_id: String, +} + +#[derive(Clone, Debug, FromRow, PartialEq)] +pub(crate) struct Notification { + pub id: i64, + pub payload: String, + pub topic: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct NotificationInput<'a> { + pub payload: &'a str, + pub topic: &'a str, +} + +// River Go stores `attempt`, `max_attempts`, and `priority` as native +// integers, so they decode as `i64`. A value `JobRow`'s field can't hold +// makes the row undecodable, like other columns River can't represent. +// JSON columns are read as bytes, since a value changed out of band might not +// even be UTF-8. +#[derive(Clone, Debug, FromRow)] +struct JobRecord { + attempt: i64, + attempted_at: Option>, + attempted_by: Option>, + created_at: DateTime, + encoded_args: Vec, + errors: Option>, + finalized_at: Option>, + id: i64, + kind: String, + max_attempts: i64, + metadata: Vec, + priority: i64, + queue: String, + scheduled_at: DateTime, + state: String, + tags: Vec, + unique_key: Option>, + unique_states: Option, +} + +impl JobRecord { + /// Decodes the row, failing if any field can't be decoded. + fn into_job(self) -> Result { + let id = self.id; + self.decode() + .map_err(|job| BackendError::InvalidRow(format!("job {id}: {}", job.error))) + } + + /// Decodes the row, keeping the fields that can be decoded when others + /// can't, like River Go. JSON columns can be changed to any shape, so + /// each decodes on its own. River Go reads the metadata as raw JSON, but a + /// [`JobRow`] can only represent an object. + fn decode(self) -> DecodedJob { + let unidentifiable = |error: String| UndecodableJob { + columns: Vec::new(), + error: format!("job {}: {error}", self.id), + row: None, + }; + let state = JobState::try_from(self.state.as_str()) + .map_err(|error| unidentifiable(error.to_string()))?; + + let mut errors = FieldErrors::default(); + // Like River Go, args that aren't valid JSON are left empty, as + // `null`, so the job isn't worked with them. + let encoded_args = errors.field_or_else( + "args", + json_text(self.encoded_args) + .and_then(|text| RawValue::from_string(text).map_err(|error| error.to_string())), + || RawValue::from_string("null".to_owned()).expect("null is valid JSON"), + ); + let attempted_by = errors.field( + "attempted_by", + self.attempted_by + .map(json_text) + .transpose() + .and_then(|text| { + decode_json_strings(text.as_deref()).map_err(|error| error.to_string()) + }), + ); + let attempt_errors = errors.field( + "errors", + self.errors.map(json_text).transpose().and_then(|text| { + text.as_deref() + .map_or_else(|| Ok(Vec::new()), AttemptError::from_json_array_lenient) + .map_err(|error| error.to_string()) + }), + ); + let metadata = errors.field( + "metadata", + json_text(self.metadata).and_then(|text| { + text.parse::() + .map_err(|error| error.to_string()) + }), + ); + let tags = errors.field( + "tags", + json_text(self.tags).and_then(|text| { + decode_json_strings(Some(&text)).map_err(|error| error.to_string()) + }), + ); + let unique_states = errors.field( + "unique_states", + self.unique_states.map(decode_unique_states).transpose(), + ); + let attempt = errors.field("attempt", i32::try_from(self.attempt)); + let max_attempts = errors.field("max_attempts", i32::try_from(self.max_attempts)); + let priority = errors.field("priority", i16::try_from(self.priority)); + errors.finish(JobRow { + id: self.id, + attempt, + attempted_at: self.attempted_at, + attempted_by, + created_at: self.created_at, + encoded_args, + errors: attempt_errors, + finalized_at: self.finalized_at, + kind: self.kind, + max_attempts, + metadata, + priority, + queue: self.queue, + scheduled_at: self.scheduled_at, + state, + tags, + unique_key: self.unique_key, + unique_states, + }) + } +} + +/// Decodes a row selected with [`JOB_COLUMNS`] on its own. +pub(crate) fn decode_job_row(row: &MySqlRow) -> DecodedJob { + JobRecord::from_row(row) + .map_err(|error| UndecodableJob { + columns: Vec::new(), + error: error.to_string(), + row: None, + })? + .decode() +} + +#[derive(Clone, Debug, FromRow)] +struct QueueRecord { + created_at: DateTime, + metadata: String, + name: String, + paused_at: Option>, + updated_at: DateTime, +} + +impl QueueRecord { + fn into_queue(self) -> Result { + let metadata: Value = serde_json::from_str(&self.metadata)?; + let metadata = metadata.as_object().cloned().ok_or_else(|| { + BackendError::InvalidRow(format!("queue {:?} metadata is not an object", self.name)) + })?; + Ok(Queue { + created_at: self.created_at, + metadata, + metadata_text: self.metadata, + name: self.name, + paused_at: self.paused_at, + updated_at: self.updated_at, + }) + } +} + +#[derive(Clone, Debug, FromRow)] +struct LeaderRecord { + elected_at: DateTime, + expires_at: DateTime, + leader_id: String, +} + +impl From for Leader { + fn from(record: LeaderRecord) -> Self { + Self { + elected_at: record.elected_at, + expires_at: record.expires_at, + leader_id: record.leader_id, + } + } +} + +/// Converts a JSON column read as bytes to text. +fn json_text(bytes: Vec) -> Result { + String::from_utf8(bytes).map_err(|error| error.to_string()) +} + +fn decode_json_or_default(encoded: Option<&str>) -> Result +where + T: serde::de::DeserializeOwned + Default, +{ + encoded.map_or_else(|| Ok(T::default()), serde_json::from_str) +} + +/// Decodes a JSON array of strings like Go's `encoding/json` decodes a +/// `[]string`: `null` is empty, as is a `null` element. +fn decode_json_strings(encoded: Option<&str>) -> Result, serde_json::Error> { + Ok( + decode_json_or_default::>>>(encoded)? + .unwrap_or_default() + .into_iter() + .map(Option::unwrap_or_default) + .collect(), + ) +} + +fn decode_unique_states(bits: i64) -> Result, String> { + let bits = u8::try_from(bits).map_err(|_| format!("value out of range for byte: {bits}"))?; + Ok(JobState::ALL + .iter() + .copied() + .filter(|state| bits & state.unique_bit() != 0) + .collect()) +} + +pub(crate) fn mysql_time(time: DateTime) -> String { + time.format("%Y-%m-%d %H:%M:%S%.6f").to_string() +} + +fn mysql_time_optional(time: Option>) -> Option { + time.map(mysql_time) +} + +pub(crate) async fn insert( + connection: &mut MySqlConnection, + params: &InsertJob<'_>, +) -> Result { + let mut metadata = params.metadata.clone(); + if let Some(nonce) = params.unique_nonce { + metadata.insert(METADATA_KEY_UNIQUE_NONCE, Value::String(nonce.to_owned()))?; + } + let result = sqlx::query( + "INSERT INTO river_job (id, args, attempt, attempted_at, attempted_by, created_at, errors, \ + finalized_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) \ + VALUES (?, CAST(? AS JSON), ?, ?, CAST(? AS JSON), COALESCE(?, UTC_TIMESTAMP(6)), CAST(? AS JSON), \ + ?, ?, ?, CAST(? AS JSON), ?, ?, COALESCE(?, UTC_TIMESTAMP(6)), ?, CAST(? AS JSON), ?, ?) \ + ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id)", + ) + .bind(params.id).bind(params.encoded_args.get()).bind(params.attempt) + .bind(mysql_time_optional(params.attempted_at)).bind(serde_json::to_string(params.attempted_by)?) + .bind(mysql_time_optional(params.created_at)).bind(serde_json::to_string(params.errors)?) + .bind(mysql_time_optional(params.finalized_at)).bind(params.kind).bind(params.max_attempts) + .bind(serde_json::to_string(&metadata)?).bind(params.priority).bind(params.queue) + .bind(mysql_time_optional(params.scheduled_at)).bind(params.state.as_str()) + .bind(serde_json::to_string(params.tags)?).bind(params.unique_key).bind(params.unique_states.map(i16::from)) + .execute(&mut *connection).await?; + let id = i64::try_from(result.last_insert_id()) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let job = get_locked(connection, id) + .await? + .ok_or(sqlx::Error::RowNotFound)?; + let unique_skipped_as_duplicate = params.unique_nonce.is_some_and(|nonce| { + job.metadata + .get::(METADATA_KEY_UNIQUE_NONCE) + .ok() + .flatten() + .as_deref() + != Some(nonce) + }); + if unique_skipped_as_duplicate + && (params.unique_key.is_none() + || params.unique_key != job.unique_key.as_deref() + || params + .unique_states + .is_none_or(|bits| bits & params.state.unique_bit() == 0) + || !job + .unique_states + .as_ref() + .is_some_and(|states| states.contains(&job.state))) + { + return Err(BackendError::InvalidRow( + "insert conflicts with a constraint other than River job uniqueness".into(), + )); + } + Ok(InsertedJob { + job, + unique_skipped_as_duplicate, + }) +} + +pub(crate) async fn claim( + connection: &mut MySqlConnection, + params: &ClaimJobs<'_>, +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + let mut query = QueryBuilder::::new("SELECT id FROM river_job WHERE queue = "); + query.push_bind(params.queue).push(" AND CAST(queue AS BINARY) = ").push_bind(params.queue) + .push(" AND state = 'available' AND scheduled_at <= ").push_bind(mysql_time(params.now)) + .push(" AND NOT EXISTS (SELECT 1 FROM river_queue WHERE name = river_job.queue AND CAST(name AS BINARY) = CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL)"); + if let Some(kinds) = params.kinds { + if kinds.is_empty() { + return Ok(Vec::new()); + } + query.push(" AND CAST(kind AS BINARY) IN ("); + let mut separated = query.separated(", "); + for kind in kinds { + separated.push_bind(kind); + } + separated.push_unseparated(")"); + } + query + .push(" ORDER BY priority, scheduled_at, id LIMIT ") + .push_bind(params.limit) + .push(" FOR UPDATE SKIP LOCKED"); + let ids = query + .build_query_scalar::() + .fetch_all(&mut *connection) + .await?; + claim_selected(connection, params, &ids).await +} + +/// Updates the selected rows while the caller holds their transaction locks. +pub(crate) async fn claim_selected( + connection: &mut MySqlConnection, + params: &ClaimJobs<'_>, + ids: &[i64], +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + let mut rows = Vec::with_capacity(ids.len()); + for id in ids { + let sql = format!( + "SELECT {JOB_COLUMNS} FROM river_job WHERE id = ? AND state = 'available' FOR UPDATE" + ); + let Some(row) = sqlx::query(AssertSqlSafe(sql)) + .bind(id) + .fetch_optional(&mut *connection) + .await? + else { + continue; + }; + // A malformed attempted_by is retained so the worker fails that row alone. + let attempted_by = row + .try_get::>, _>("attempted_by")? + .as_deref() + .map(serde_json::from_slice::>>>) + .transpose(); + let attempted_by = attempted_by + .ok() + .map(|values| { + let mut values = values.flatten().unwrap_or_default(); + let keep = + usize::try_from(params.max_attempted_by.saturating_sub(1)).unwrap_or_default(); + values.drain(..values.len().saturating_sub(keep)); + values.push(Some(params.client_id.to_owned())); + serde_json::to_string(&values) + }) + .transpose()?; + sqlx::query("UPDATE river_job SET attempt = attempt + 1, attempted_at = ?, attempted_by = COALESCE(CAST(? AS JSON), attempted_by), state = 'running' WHERE id = ?") + .bind(mysql_time(params.now)).bind(attempted_by).bind(id).execute(&mut *connection).await?; + if let Some(row) = get_raw(connection, *id, false).await? { + rows.push(decode_job_row(&row)); + } + } + Ok(rows) +} + +async fn get_raw( + connection: &mut MySqlConnection, + id: i64, + lock: bool, +) -> Result, BackendError> { + let sql = format!( + "SELECT {JOB_COLUMNS} FROM river_job WHERE id = ?{}", + if lock { " FOR UPDATE" } else { "" } + ); + Ok(sqlx::query(AssertSqlSafe(sql)) + .bind(id) + .fetch_optional(&mut *connection) + .await?) +} + +pub(crate) async fn get( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + get_raw(connection, id, false) + .await? + .as_ref() + .map(|row| JobRecord::from_row(row)?.into_job()) + .transpose() +} + +pub(crate) async fn get_locked( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + get_raw(connection, id, true) + .await? + .as_ref() + .map(|row| JobRecord::from_row(row)?.into_job()) + .transpose() +} + +/// Merge objects shallowly, retaining JSON null values as Postgres does. +fn push_metadata( + query: &mut QueryBuilder, + updates: &Map, +) -> Result<(), BackendError> { + if updates.is_empty() { + query.push("metadata"); + return Ok(()); + } + query.push("JSON_SET(metadata"); + for (key, value) in updates { + query + .push(", ") + .push_bind(format!("$.{}", serde_json::to_string(key)?)) + .push(", CAST(") + .push_bind(serde_json::to_string(value)?) + .push(" AS JSON)"); + } + query.push(")"); + Ok(()) +} + +pub(crate) async fn cancel( + connection: &mut MySqlConnection, + id: i64, + now: DateTime, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.finalized_at.is_some() + || matches!( + row.state, + JobState::Cancelled | JobState::Completed | JobState::Discarded + ) + { + return Ok(None); + } + sqlx::query("UPDATE river_job SET finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE ? END, metadata = JSON_SET(metadata, '$.cancel_attempted_at', ?), state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END WHERE id = ?") + .bind(mysql_time(now)).bind(go_time_json(now)).bind(id).execute(&mut *connection).await?; + get(connection, id).await +} + +pub(crate) async fn delete( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.state == JobState::Running { + return Ok(None); + } + sqlx::query("DELETE FROM river_job WHERE id = ?") + .bind(id) + .execute(connection) + .await?; + Ok(Some(row)) +} + +pub(crate) async fn retry( + connection: &mut MySqlConnection, + id: i64, + now: DateTime, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.state == JobState::Running + || (row.state == JobState::Available && row.scheduled_at <= now) + { + return Ok(None); + } + sqlx::query("UPDATE river_job SET state = 'available', finalized_at = NULL, scheduled_at = ?, max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END WHERE id = ?") + .bind(mysql_time(now)).bind(id).execute(&mut *connection).await?; + get(connection, id).await +} + +pub(crate) async fn complete( + connection: &mut MySqlConnection, + params: &CompleteJob<'_>, +) -> Result, BackendError> { + complete_decoded(connection, params) + .await? + .map(|row| row.map_err(|job| BackendError::InvalidRow(job.error))) + .transpose() +} + +pub(crate) async fn complete_decoded( + connection: &mut MySqlConnection, + params: &CompleteJob<'_>, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, params.id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? != "running" { + return Ok(None); + } + let metadata: Value = serde_json::from_slice(&row.try_get::, _>("metadata")?)?; + let cancel = matches!( + params.state, + JobState::Available | JobState::Retryable | JobState::Scheduled + ) && metadata + .as_object() + .is_some_and(|metadata| metadata.contains_key("cancel_attempted_at")); + let mut query = QueryBuilder::::new("UPDATE river_job SET state = "); + query.push_bind(if cancel { + "cancelled" + } else { + params.state.as_str() + }); + if cancel { + query + .push(", finalized_at = ") + .push_bind(mysql_time(params.now)); + } else { + if let Some(attempt) = params.attempt { + query.push(", attempt = ").push_bind(attempt); + } + if let Some(time) = params.finalized_at { + query.push(", finalized_at = ").push_bind(mysql_time(time)); + } + if let Some(time) = params.scheduled_at { + query.push(", scheduled_at = ").push_bind(mysql_time(time)); + } + } + if let Some(error) = params.error { + // This expression has one placeholder; bind through the query builder. + let (before, after) = ERRORS_APPENDED + .split_once('?') + .expect("one error parameter"); + query + .push(", errors = ") + .push(before) + .push_bind(serde_json::to_string(error)?) + .push(after); + } + if let Some(metadata) = params.metadata_updates { + query.push(", metadata = "); + push_metadata(&mut query, metadata)?; + } + query.push(" WHERE id = ").push_bind(params.id); + query.build().execute(&mut *connection).await?; + Ok(get_raw(connection, params.id, false) + .await? + .as_ref() + .map(decode_job_row)) +} + +pub(crate) async fn merge_metadata_if_not_running( + connection: &mut MySqlConnection, + id: i64, + updates: &Map, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? == "running" { + return Ok(None); + } + let mut query = QueryBuilder::::new("UPDATE river_job SET metadata = "); + push_metadata(&mut query, updates)?; + query.push(" WHERE id = ").push_bind(id); + query.build().execute(&mut *connection).await?; + Ok(get_raw(connection, id, false) + .await? + .as_ref() + .map(decode_job_row)) +} + +pub(crate) async fn update( + connection: &mut MySqlConnection, + id: i64, + updates: &Map, +) -> Result, BackendError> { + if get_raw(connection, id, true).await?.is_none() { + return Ok(None); + } + let mut query = QueryBuilder::::new("UPDATE river_job SET metadata = "); + push_metadata(&mut query, updates)?; + query.push(" WHERE id = ").push_bind(id); + query.build().execute(&mut *connection).await?; + get(connection, id).await +} + +#[allow( + clippy::too_many_lines, + reason = "the bound-only query builder keeps all list filters and keyset ordering auditable" +)] +pub(crate) async fn list( + connection: &mut MySqlConnection, + params: &ListJobs<'_>, +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + + let mut query = + QueryBuilder::::new(format!("SELECT {JOB_COLUMNS} FROM river_job WHERE true")); + if let Some(after) = params.keyset.after_sql() { + query.push(" AND "); + for part in after { + match part { + JobListSqlPart::AfterId => { + query.push_bind(params.keyset.after_id().expect("cursor has an ID")); + } + JobListSqlPart::AfterTime => { + query.push_bind(mysql_time( + params.keyset.after_time().expect("cursor has a time"), + )); + } + JobListSqlPart::Sql(sql) => { + query.push(sql); + } + } + } + } + if params.exclude_running { + query.push(" AND state != 'running'"); + } + if !params.ids.is_empty() { + query.push(" AND id IN ("); + let mut separated = query.separated(", "); + for id in params.ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + } + if !params.kinds.is_empty() { + query.push(" AND CAST(kind AS BINARY) IN ("); + let mut separated = query.separated(", "); + for kind in params.kinds { + separated.push_bind(kind); + } + separated.push_unseparated(")"); + } + if !params.priorities.is_empty() { + query.push(" AND priority IN ("); + let mut separated = query.separated(", "); + for priority in params.priorities { + separated.push_bind(priority); + } + separated.push_unseparated(")"); + } + if !params.queues.is_empty() { + query.push(" AND CAST(queue AS BINARY) IN ("); + let mut separated = query.separated(", "); + for queue in params.queues { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + if !params.states.is_empty() { + query.push(" AND state IN ("); + let mut separated = query.separated(", "); + for state in params.states { + separated.push_bind(state.as_str()); + } + separated.push_unseparated(")"); + } + if let Some(metadata) = params.metadata { + query.push(" AND "); + push_contains_value( + &mut query, + "metadata", + &Value::Object(metadata.clone()), + &mut 0, + )?; + } + for tag in params.tags_all { + query + .push(" AND JSON_CONTAINS(tags, JSON_QUOTE(") + .push_bind(tag) + .push("))"); + } + if !params.tags_any.is_empty() { + query.push(" AND EXISTS (SELECT 1 FROM JSON_TABLE(tags, '$[*]' COLUMNS(value VARCHAR(255) PATH '$')) AS tag WHERE CAST(value AS BINARY) IN ("); + let mut separated = query.separated(", "); + for tag in params.tags_any { + separated.push_bind(tag); + } + separated.push_unseparated("))"); + } + query + .push(" ORDER BY ") + .push(mysql_order(¶ms.keyset.order_sql())); + query.push(" LIMIT ").push_bind(params.limit); + + let records = query + .build_query_as::() + .fetch_all(&mut *connection) + .await?; + records.into_iter().map(JobRecord::into_job).collect() +} + +fn mysql_order(order: &str) -> String { + let field = order.split_whitespace().next().expect("order has a field"); + if order.contains("NULLS LAST") { + format!("{field} IS NULL ASC, {}", order.replace(" NULLS LAST", "")) + } else if order.contains("NULLS FIRST") { + format!( + "{field} IS NULL DESC, {}", + order.replace(" NULLS FIRST", "") + ) + } else { + order.to_owned() + } +} + +// JSON_CONTAINS flattens nested arrays; Postgres containment preserves each +// level. JSON_TABLE traverses each level explicitly. Scalar subqueries avoid +// MySQL 8 semijoin rewrites losing nested correlations. +fn push_contains_value( + query: &mut QueryBuilder, + source: &str, + fragment: &Value, + aliases: &mut usize, +) -> Result<(), BackendError> { + match fragment { + Value::Object(object) => { + query.push(format!("(JSON_TYPE({source}) = 'OBJECT'")); + for (key, value) in object { + let alias = format!("contains_{aliases}"); + *aliases += 1; + query + .push(format!( + " AND (SELECT 1 FROM JSON_TABLE(JSON_EXTRACT({source}, " + )) + .push_bind(format!("$.{}", serde_json::to_string(key)?)) + .push(format!( + "), '$' COLUMNS(value JSON PATH '$')) AS {alias} WHERE " + )); + push_contains_value(query, &format!("{alias}.value"), value, aliases)?; + query.push(" LIMIT 1) IS NOT NULL"); + } + query.push(")"); + } + Value::Array(array) => { + query.push(format!("(JSON_TYPE({source}) = 'ARRAY'")); + for value in array { + let alias = format!("contains_{aliases}"); + *aliases += 1; + query.push(format!(" AND (SELECT 1 FROM JSON_TABLE({source}, '$[*]' COLUMNS(value JSON PATH '$')) AS {alias} WHERE ")); + push_contains_value(query, &format!("{alias}.value"), value, aliases)?; + query.push(" LIMIT 1) IS NOT NULL"); + } + query.push(")"); + } + Value::Null => { + query.push(format!( + "({source} IS NULL OR JSON_TYPE({source}) = 'NULL')" + )); + } + value => { + query.push(format!("(JSON_TYPE({source}) NOT IN ('ARRAY', 'OBJECT') AND JSON_CONTAINS({source}, CAST(")) + .push_bind(serde_json::to_string(value)?).push(" AS JSON)))"); + } + } + Ok(()) +} + +pub(crate) async fn queue_upsert( + connection: &mut MySqlConnection, + name: &str, + metadata: &Map, + paused_at: Option>, + now: DateTime, +) -> Result { + sqlx::query("INSERT INTO river_queue (created_at, metadata, name, paused_at, updated_at) VALUES (?, CAST(? AS JSON), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?") + .bind(mysql_time(now)).bind(serde_json::to_string(metadata)?).bind(name) + .bind(mysql_time_optional(paused_at)).bind(mysql_time(now)).bind(mysql_time(now)) + .execute(&mut *connection).await?; + queue_get(connection, name) + .await? + .ok_or_else(|| sqlx::Error::RowNotFound.into()) +} + +pub(crate) async fn queue_get( + connection: &mut MySqlConnection, + name: &str, +) -> Result, BackendError> { + sqlx::query_as::<_, QueueRecord>(AssertSqlSafe(format!( + "SELECT {QUEUE_COLUMNS} FROM river_queue WHERE name = ? AND CAST(name AS BINARY) = ?" + ))) + .bind(name) + .bind(name) + .fetch_optional(connection) + .await? + .map(QueueRecord::into_queue) + .transpose() +} + +pub(crate) async fn queue_list( + connection: &mut MySqlConnection, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + sqlx::query_as::<_, QueueRecord>(AssertSqlSafe(format!( + "SELECT {QUEUE_COLUMNS} FROM river_queue ORDER BY name LIMIT ?" + ))) + .bind(limit) + .fetch_all(connection) + .await? + .into_iter() + .map(QueueRecord::into_queue) + .collect() +} + +pub(crate) async fn queue_pause( + connection: &mut MySqlConnection, + name: &str, + now: DateTime, +) -> Result, BackendError> { + queue_set_paused(connection, name, Some(now), now).await +} + +pub(crate) async fn queue_resume( + connection: &mut MySqlConnection, + name: &str, + now: DateTime, +) -> Result, BackendError> { + queue_set_paused(connection, name, None, now).await +} + +async fn queue_set_paused( + connection: &mut MySqlConnection, + name: &str, + paused_at: Option>, + now: DateTime, +) -> Result, BackendError> { + let names = sqlx::query_scalar::<_, String>("SELECT name FROM river_queue WHERE (? = '*' OR CAST(name AS BINARY) = ?) ORDER BY name FOR UPDATE") + .bind(name).bind(name).fetch_all(&mut *connection).await?; + let mut queues = Vec::with_capacity(names.len()); + for name in names { + sqlx::query("UPDATE river_queue SET updated_at = CASE WHEN (? AND paused_at IS NULL) OR (NOT ? AND paused_at IS NOT NULL) THEN ? ELSE updated_at END, paused_at = CASE WHEN ? THEN COALESCE(paused_at, ?) ELSE NULL END WHERE name = ?") + .bind(paused_at.is_some()).bind(paused_at.is_some()).bind(mysql_time(now)) + .bind(paused_at.is_some()).bind(mysql_time_optional(paused_at)).bind(&name).execute(&mut *connection).await?; + if let Some(queue) = queue_get(connection, &name).await? { + queues.push(queue); + } + } + Ok(queues) +} + +pub(crate) async fn queue_update( + connection: &mut MySqlConnection, + name: &str, + metadata: Option<&Map>, + now: DateTime, +) -> Result, BackendError> { + sqlx::query("UPDATE river_queue SET metadata = COALESCE(CAST(? AS JSON), metadata), updated_at = ? WHERE name = ? AND CAST(name AS BINARY) = ?") + .bind(metadata.map(serde_json::to_string).transpose()?).bind(mysql_time(now)).bind(name).bind(name).execute(&mut *connection).await?; + queue_get(connection, name).await +} + +pub(crate) async fn queue_delete_expired( + connection: &mut MySqlConnection, + updated_before: DateTime, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + let names = sqlx::query_scalar::<_, String>("SELECT name FROM river_queue WHERE updated_at < ? ORDER BY name LIMIT ? FOR UPDATE SKIP LOCKED") + .bind(mysql_time(updated_before)).bind(limit).fetch_all(&mut *connection).await?; + for name in &names { + sqlx::query("DELETE FROM river_queue WHERE name = ?") + .bind(name) + .execute(&mut *connection) + .await?; + } + Ok(names) +} + +pub(crate) async fn leader_elect( + connection: &mut MySqlConnection, + leader_id: &str, + now: DateTime, + ttl: Duration, +) -> Result, BackendError> { + let expires_at = now + + chrono::Duration::from_std(ttl) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let result = sqlx::query("INSERT INTO river_leader (name, leader_id, elected_at, expires_at) VALUES ('default', ?, ?, ?)") + .bind(leader_id).bind(mysql_time(now)).bind(mysql_time(expires_at)).execute(&mut *connection).await; + match result { + Ok(_) => leader_get(connection).await, + Err(error) + if error + .as_database_error() + .is_some_and(sqlx::error::DatabaseError::is_unique_violation) => + { + Ok(None) + } + Err(error) => Err(error.into()), + } +} + +async fn leader_get(connection: &mut MySqlConnection) -> Result, BackendError> { + Ok(sqlx::query_as::<_, LeaderRecord>("SELECT elected_at, expires_at, leader_id FROM river_leader WHERE name = 'default' FOR UPDATE") + .fetch_optional(connection).await?.map(Leader::from)) +} + +pub(crate) async fn leader_reelect( + connection: &mut MySqlConnection, + leader_id: &str, + elected_at: DateTime, + now: DateTime, + ttl: Duration, +) -> Result, BackendError> { + let expires_at = now + + chrono::Duration::from_std(ttl) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let result = sqlx::query("UPDATE river_leader SET expires_at = ? WHERE name = 'default' AND CAST(leader_id AS BINARY) = ? AND elected_at = ? AND expires_at >= ?") + .bind(mysql_time(expires_at)).bind(leader_id).bind(mysql_time(elected_at)).bind(mysql_time(now)).execute(&mut *connection).await?; + if result.rows_affected() == 0 { + return Ok(None); + } + leader_get(connection).await +} + +pub(crate) async fn leader_resign( + connection: &mut MySqlConnection, + leader_id: &str, + elected_at: DateTime, +) -> Result { + Ok(sqlx::query("DELETE FROM river_leader WHERE name = 'default' AND CAST(leader_id AS BINARY) = ? AND elected_at = ?") + .bind(leader_id).bind(mysql_time(elected_at)).execute(connection).await?.rows_affected() > 0) +} + +pub(crate) async fn leader_delete_expired( + connection: &mut MySqlConnection, + now: DateTime, +) -> Result { + Ok(sqlx::query("DELETE FROM river_leader WHERE expires_at < ?") + .bind(mysql_time(now)) + .execute(connection) + .await? + .rows_affected()) +} + +pub(crate) async fn notification_insert( + connection: &mut MySqlConnection, + notifications: &[NotificationInput<'_>], +) -> Result { + if notifications.is_empty() { + return Ok(0); + } + let mut query = QueryBuilder::::new("INSERT INTO river_notification (payload, topic) "); + query.push_values(notifications, |mut row, notification| { + row.push_bind(notification.payload) + .push_bind(notification.topic); + }); + Ok(query.build().execute(connection).await?.rows_affected()) +} + +// Ordered locking reads wait for earlier uncommitted IDs. MAX(id), snapshot +// reads and SKIP LOCKED can advance the cursor past a later commit forever. +pub(crate) async fn notification_get_after( + connection: &mut MySqlConnection, + after_id: i64, + topics: &[&str], + limit: i64, +) -> Result, BackendError> { + if limit <= 0 || topics.is_empty() { + return Ok(Vec::new()); + } + Ok(sqlx::query_as::<_, Notification>("SELECT id, payload, topic FROM river_notification FORCE INDEX (PRIMARY) WHERE id > ? ORDER BY id LIMIT ? FOR SHARE") + .bind(after_id).bind(limit).fetch_all(connection).await?) +} + +pub(crate) async fn notification_last_id( + connection: &mut MySqlConnection, +) -> Result { + // Read every ID to lock lower outstanding inserts before establishing the cursor. + Ok(sqlx::query_scalar::<_, i64>( + "SELECT id FROM river_notification FORCE INDEX (PRIMARY) ORDER BY id FOR UPDATE", + ) + .fetch_all(connection) + .await? + .last() + .copied() + .unwrap_or_default()) +} + +#[derive(Debug, Default)] +pub(crate) struct NotificationListener { + /// ID of the last row read, or `None` before connecting. + last_id: Option, + /// Rows read but not yet delivered. + pending: VecDeque, + /// Subscribed topics and the ID after which each one's rows are + /// delivered. + topics: HashMap, +} + +impl NotificationListener { + /// Starts reading after the outbox's current maximum ID. + pub(crate) async fn connect( + &mut self, + connection: &mut MySqlConnection, + ) -> Result<(), BackendError> { + let mut tx = connection.begin().await?; + let last_id = notification_last_id(&mut tx).await?; + tx.commit().await?; + self.last_id = Some(last_id); + Ok(()) + } + + /// Whether the initial outbox cursor has been established. + pub(crate) const fn is_connected(&self) -> bool { + self.last_id.is_some() + } + + /// Subscribes to `topic`, delivering only rows written from now on. + /// Subscribing to a topic that's already subscribed changes nothing. + pub(crate) async fn listen( + &mut self, + connection: &mut MySqlConnection, + topic: &str, + ) -> Result<(), BackendError> { + if self.topics.contains_key(topic) { + return Ok(()); + } + // Only this topic's starting ID advances. Advancing the shared cursor + // could skip undelivered rows of topics that are already subscribed. + let mut tx = connection.begin().await?; + let last_id = notification_last_id(&mut tx).await?; + tx.commit().await?; + // Cleanup may have deleted rows that are still buffered locally. + let start_id = last_id.max(self.last_id.unwrap_or_default()); + self.topics.insert(topic.to_owned(), start_id); + Ok(()) + } + + /// Returns the next notification on a subscribed topic, or `None` once + /// the outbox has no newer rows on subscribed topics. + pub(crate) async fn next( + &mut self, + connection: &mut MySqlConnection, + ) -> Result, BackendError> { + loop { + if self.pending.is_empty() { + let Some(after_id) = self.last_id else { + return Ok(None); + }; + let topics = self.topics.keys().map(String::as_str).collect::>(); + // Lock each bounded scan in its own transaction, then release + // its locks before dispatching notifications. Cursor and buffer + // changes become visible only after a successful commit. + let mut tx = connection.begin().await?; + let notifications = + notification_get_after(&mut tx, after_id, &topics, NOTIFICATION_BATCH_SIZE) + .await?; + tx.commit().await?; + let Some(last) = notifications.last() else { + return Ok(None); + }; + self.last_id = Some(last.id); + self.pending.extend(notifications); + } + // Subscriptions can change between reading a batch and delivering + // it. + while let Some(notification) = self.pending.pop_front() { + if self + .topics + .get(¬ification.topic) + .is_some_and(|start_id| notification.id > *start_id) + { + return Ok(Some(notification)); + } + } + } + } +} + +pub(crate) async fn notification_cleanup( + connection: &mut MySqlConnection, + created_before: DateTime, + limit: i64, +) -> Result { + if limit <= 0 { + return Ok(0); + } + Ok(sqlx::query( + "DELETE FROM river_notification WHERE created_at < ? ORDER BY created_at, id LIMIT ?", + ) + .bind(mysql_time(created_before)) + .bind(limit) + .execute(&mut *connection) + .await? + .rows_affected()) +} + +pub(crate) async fn stuck_jobs( + connection: &mut MySqlConnection, + after_id: i64, + attempted_before: DateTime, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + let sql = format!( + r#" + SELECT {JOB_COLUMNS} + FROM river_job + WHERE state = 'running' + AND id > ? + AND attempted_at < ? + ORDER BY id ASC + LIMIT ? + "# + ); + let records = sqlx::query(AssertSqlSafe(sql)) + .bind(after_id) + .bind(mysql_time(attempted_before)) + .bind(limit) + .fetch_all(&mut *connection) + .await?; + Ok(tolerant_rows(&records)) +} + +/// Loads exactly the rescue candidates selected by an exact-version +/// extension. The enclosing transaction protects the subsequent state +/// transition. +pub(crate) async fn jobs_by_ids( + connection: &mut MySqlConnection, + ids: &[i64], +) -> Result, BackendError> { + if ids.is_empty() { + return Ok(Vec::new()); + } + let mut query = + QueryBuilder::::new(format!("SELECT {JOB_COLUMNS} FROM river_job WHERE id IN (")); + { + let mut separated = query.separated(", "); + for id in ids { + separated.push_bind(id); + } + } + query.push(") ORDER BY id ASC"); + let records = query.build().fetch_all(&mut *connection).await?; + Ok(tolerant_rows(&records)) +} + +/// Decodes rows for the rescuer, which like River Go's `JobGetStuck` +/// tolerates undecodable fields so a job stranded by one can be rescued. +fn tolerant_rows(records: &[MySqlRow]) -> Vec { + records + .iter() + .filter_map(|record| tolerant_row(decode_job_row(record))) + .collect() +} + +pub(crate) async fn cleanup_jobs( + connection: &mut MySqlConnection, + params: &CleanupJobs<'_>, +) -> Result { + let horizons = [ + ("cancelled", params.cancelled_before), + ("completed", params.completed_before), + ("discarded", params.discarded_before), + ]; + if params.limit <= 0 || horizons.iter().all(|(_, before)| before.is_none()) { + return Ok(0); + } + let mut query = QueryBuilder::::new("SELECT id FROM river_job WHERE ("); + let mut first = true; + for (state, before) in horizons { + let Some(before) = before else { + continue; + }; + if !first { + query.push(" OR "); + } + first = false; + query + .push("(state = ") + .push_bind(state) + .push(" AND finalized_at < ") + .push_bind(mysql_time(before)) + .push(")"); + } + query.push(")"); + if !params.queues_excluded.is_empty() { + query.push(" AND CAST(queue AS BINARY) NOT IN ("); + let mut separated = query.separated(", "); + for queue in params.queues_excluded { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + if let Some(queues_included) = params.queues_included { + if queues_included.is_empty() { + return Ok(0); + } + query.push(" AND CAST(queue AS BINARY) IN ("); + let mut separated = query.separated(", "); + for queue in queues_included { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + query + .push(" ORDER BY id ASC LIMIT ") + .push_bind(params.limit) + .push(" FOR UPDATE SKIP LOCKED"); + let mut tx = connection.begin().await?; + let ids: Vec = query.build_query_scalar().fetch_all(&mut *tx).await?; + let deleted = if ids.is_empty() { + 0 + } else { + let mut delete = QueryBuilder::::new( + "DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM river_job WHERE id IN (", + ); + let mut separated = delete.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + delete.build().execute(&mut *tx).await?.rows_affected() + }; + tx.commit().await?; + Ok(deleted) +} + +/// A due job the scheduler may make available. +#[derive(Clone, Debug, FromRow)] +pub(crate) struct ScheduleCandidate { + pub id: i64, + pub unique_key: Option>, +} + +/// Selects due retryable/scheduled jobs in scheduler order. Scheduling is a +/// multi-step MySQL operation: callers keep a write transaction open while +/// checking unique collisions and applying the transitions below. +/// +/// Like River Go, only the columns scheduling needs are selected, so a job +/// whose row can't be decoded, like one whose attempt failed because a JSON +/// column isn't valid JSON, doesn't fail scheduling for every other job. The +/// transitions below return such a job's row with the undecodable fields left +/// empty. +pub(crate) async fn schedule_candidates( + connection: &mut MySqlConnection, + now: DateTime, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + Ok(sqlx::query_as::<_, ScheduleCandidate>( + r#" + SELECT id, unique_key + FROM river_job + WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= ? + ORDER BY priority ASC, scheduled_at ASC, id ASC + LIMIT ? FOR UPDATE SKIP LOCKED + "#, + ) + .bind(mysql_time(now)) + .bind(limit) + .fetch_all(&mut *connection) + .await?) +} + +pub(crate) async fn schedule_has_unique_collision( + connection: &mut MySqlConnection, + id: i64, + unique_key: &[u8], +) -> Result { + let exists = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM river_job + WHERE id != ? + AND unique_key = ? + AND unique_states IS NOT NULL + AND CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + ) + "#, + ) + .bind(id) + .bind(unique_key) + .fetch_one(&mut *connection) + .await?; + Ok(exists) +} + +pub(crate) async fn rescue( + connection: &mut MySqlConnection, + params: &RescueJob<'_>, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, params.id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? != "running" + || row + .try_get::>, _>("attempted_at")? + .is_none_or(|time| time >= params.stuck_horizon) + { + return Ok(None); + } + let sql = format!( + "UPDATE river_job SET errors = {ERRORS_APPENDED}, finalized_at = ?, scheduled_at = ?, \ + metadata = JSON_SET(metadata, '$.\"river:rescue_count\"', CASE WHEN JSON_TYPE(JSON_EXTRACT(metadata, '$.\"river:rescue_count\"')) IN ('INTEGER', 'DOUBLE', 'DECIMAL') THEN JSON_EXTRACT(metadata, '$.\"river:rescue_count\"') + 1 ELSE 1 END), state = ? WHERE id = ?" + ); + sqlx::query(AssertSqlSafe(sql)) + .bind(serde_json::to_string(params.error)?) + .bind(mysql_time_optional(params.finalized_at)) + .bind(mysql_time(params.scheduled_at)) + .bind(params.state.as_str()) + .bind(params.id) + .execute(&mut *connection) + .await?; + Ok(get_raw(connection, params.id, false) + .await? + .as_ref() + .and_then(|row| tolerant_row(decode_job_row(row)))) +} + +pub(crate) async fn schedule_set_available( + connection: &mut MySqlConnection, + ids: &[i64], +) -> Result, BackendError> { + let mut available = Vec::with_capacity(ids.len()); + for id in ids { + match sqlx::query("UPDATE river_job SET state = 'available' WHERE id = ? AND state IN ('scheduled', 'retryable')").bind(id).execute(&mut *connection).await { + Ok(result) if result.rows_affected() > 0 => { + if let Some(row) = get_raw(connection, *id, false).await?.as_ref().and_then(|row| tolerant_row(decode_job_row(row))) { available.push(row); } + } + Ok(_) => {} + Err(error) if error.as_database_error().is_some_and(sqlx::error::DatabaseError::is_unique_violation) => { + // A concurrent unique insert may win after the scheduler's + // collision check. InnoDB rolls back this statement only. + schedule_discard_conflicts(connection, &[*id], Utc::now()).await?; + } + Err(error) => return Err(error.into()), + } + } + Ok(available) +} + +pub(crate) async fn schedule_discard_conflicts( + connection: &mut MySqlConnection, + ids: &[i64], + now: DateTime, +) -> Result, BackendError> { + for id in ids { + sqlx::query("UPDATE river_job SET metadata = JSON_SET(metadata, '$.unique_key_conflict', 'scheduler_discarded'), finalized_at = ?, state = 'discarded' WHERE id = ?") + .bind(mysql_time(now)).bind(id).execute(&mut *connection).await?; + } + jobs_by_ids(connection, ids).await +} + +#[cfg(all(test, river_mysql_tests))] +mod tests; diff --git a/rust/riverqueue/src/database/mysql/connection.rs b/rust/riverqueue/src/database/mysql/connection.rs new file mode 100644 index 000000000..4d182f19d --- /dev/null +++ b/rust/riverqueue/src/database/mysql/connection.rs @@ -0,0 +1,92 @@ +//! Pooled MySQL connections that are discarded when an operation is cancelled. + +use std::ops::{Deref, DerefMut}; + +use sqlx::{ + MySql, MySqlConnection, MySqlPool, mysql::MySqlTransactionManager, pool::PoolConnection, +}; +use sqlx_core::transaction::TransactionManager; + +/// Returns a connection to the pool only after its operation finishes. +/// +/// SQLx's MySQL protocol recovery can hang if a future is dropped while a +/// statement is being prepared. Returning that connection to the pool then +/// leaves its release-time ping waiting forever, including during pool close. +/// Closing the socket on cancellation also rolls back any open transaction. +#[derive(Debug)] +pub(crate) struct Connection(Option>); + +impl Connection { + pub(crate) async fn acquire(pool: &MySqlPool) -> Result { + Ok(Self(Some(pool.acquire().await?))) + } + + /// Returns a connection whose operation completed to the caller's pool. + pub(crate) fn release(mut self) { + drop(self.0.take()); + } +} + +impl Deref for Connection { + type Target = MySqlConnection; + + fn deref(&self) -> &Self::Target { + self.0.as_deref().expect("connection has not been released") + } +} + +impl DerefMut for Connection { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + .as_deref_mut() + .expect("connection has not been released") + } +} + +impl Drop for Connection { + fn drop(&mut self) { + // A graceful close or the pool's release-time ping could wait on the + // abandoned statement. Detaching and dropping closes the socket instead. + drop(self.0.take().map(PoolConnection::detach)); + } +} + +/// A River-owned transaction that discards its connection unless committed. +#[derive(Debug)] +pub(crate) struct Transaction(Connection); + +impl Transaction { + pub(crate) async fn begin(pool: &MySqlPool) -> Result { + let mut connection = Connection::acquire(pool).await?; + // Use SQLx's transaction manager so extension savepoints see the + // correct transaction depth, just as with a SQLx Transaction. + MySqlTransactionManager::begin( + &mut connection, + Some(sqlx::SqlStr::from_static( + "SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN", + )), + ) + .await?; + Ok(Self(connection)) + } + + pub(crate) async fn commit(mut self) -> Result<(), sqlx::Error> { + MySqlTransactionManager::commit(&mut self.0).await?; + self.0.release(); + Ok(()) + } +} + +impl Deref for Transaction { + type Target = MySqlConnection; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Transaction { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/rust/riverqueue/src/database/mysql/tests.rs b/rust/riverqueue/src/database/mysql/tests.rs new file mode 100644 index 000000000..9c2fe1532 --- /dev/null +++ b/rust/riverqueue/src/database/mysql/tests.rs @@ -0,0 +1,857 @@ +use std::{ + collections::HashSet, + str::FromStr, + sync::atomic::{AtomicU64, Ordering}, +}; + +use riverqueue_migrate::{Direction, MigrateOpts, MySqlMigrator}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sqlx::{ + MySqlPool, + mysql::{MySqlConnectOptions, MySqlPoolOptions}, +}; + +use super::*; +use crate::{Client, InsertOpts, JobArgs, JobListParams, UniqueOpts}; + +mod conformance; + +#[derive(Debug, Deserialize, JobArgs, Serialize)] +#[river(kind = "mysql_test")] +struct Args { + value: i64, +} + +struct TestDatabase { + admin: MySqlPool, + client: Client, + name: String, + pool: MySqlPool, +} + +impl TestDatabase { + async fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let url = std::env::var("RIVER_RUST_MYSQL_URL") + .expect("RIVER_RUST_MYSQL_URL is required for MySQL tests"); + let options = MySqlConnectOptions::from_str(&url).unwrap(); + let admin = MySqlPoolOptions::new() + .max_connections(1) + .connect_with(options.clone()) + .await + .unwrap(); + let name = format!( + "river_rust_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + sqlx::query(AssertSqlSafe(format!("CREATE DATABASE `{name}`"))) + .execute(&admin) + .await + .unwrap(); + let pool = MySqlPoolOptions::new() + .max_connections(12) + .connect_with(options.database(&name)) + .await + .unwrap(); + MySqlMigrator::new(pool.clone()).migrate_up().await.unwrap(); + let client = Client::builder(pool.clone()).build().unwrap(); + Self { + admin, + client, + name, + pool, + } + } + + async fn close(self) { + tokio::time::timeout(Duration::from_secs(10), self.pool.close()) + .await + .expect("MySQL test pool did not close"); + sqlx::query(AssertSqlSafe(format!("DROP DATABASE `{}`", self.name))) + .execute(&self.admin) + .await + .unwrap(); + self.admin.close().await; + } +} + +#[tokio::test] +async fn cancelled_operations_do_not_block_pool_close() { + use crate::storage::{Access, Session}; + + for access in [Access::Autocommit, Access::Transaction] { + let db = TestDatabase::new().await; + // Keep the blocker outside the pool so closing the pool only waits + // for the operation under test, while its statement is still blocked. + let mut blocker = db.pool.acquire().await.unwrap().detach(); + sqlx::raw_sql("LOCK TABLES river_job WRITE") + .execute(&mut blocker) + .await + .unwrap(); + + let mut session = Session::begin(&db.client.inner.database, access) + .await + .unwrap(); + let client = db.client.clone(); + let query = tokio::spawn(async move { + session.storage(&client.inner).job_get(1).await.unwrap(); + session.commit().await.unwrap(); + }); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM performance_schema.threads \ + WHERE PROCESSLIST_DB = ? AND PROCESSLIST_STATE = 'Waiting for table metadata lock'", + ) + .bind(&db.name) + .fetch_one(&db.admin) + .await + .unwrap(); + if waiting > 0 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("query did not block while preparing its statement"); + + query.abort(); + assert!(query.await.unwrap_err().is_cancelled()); + let closed = tokio::time::timeout(Duration::from_secs(2), db.pool.close()).await; + sqlx::raw_sql("UNLOCK TABLES") + .execute(&mut blocker) + .await + .unwrap(); + drop(blocker); + db.close().await; + closed.expect("pool close waited for a cancelled query's table lock"); + } +} + +#[tokio::test] +async fn cleanup_skips_locked_jobs() { + let db = TestDatabase::new().await; + let running = db + .client + .insert(Args { value: 1 }) + .await + .unwrap() + .job + .row + .id; + let locked = db + .client + .insert(Args { value: 2 }) + .await + .unwrap() + .job + .row + .id; + let expired = db + .client + .insert(Args { value: 3 }) + .await + .unwrap() + .job + .row + .id; + for value in 4..1003 { + db.client.insert(Args { value }).await.unwrap(); + } + sqlx::query("UPDATE river_job SET state='completed', finalized_at='1970-01-01' WHERE id >= ?") + .bind(locked) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("UPDATE river_job SET state='running' WHERE id=?") + .bind(running) + .execute(&db.pool) + .await + .unwrap(); + let mut lock = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + sqlx::query("SELECT id FROM river_job WHERE id IN (?, ?) FOR UPDATE") + .bind(running) + .bind(locked) + .fetch_all(&mut *lock) + .await + .unwrap(); + let mut connection = db.pool.acquire().await.unwrap(); + let deleted = tokio::time::timeout( + Duration::from_secs(2), + cleanup_jobs( + &mut connection, + &CleanupJobs { + cancelled_before: None, + completed_before: Some(Utc::now()), + discarded_before: None, + limit: 1000, + queues_excluded: &[], + queues_included: None, + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deleted, 1000); + assert!(db.client.jobs().get(expired).await.is_err()); + assert_eq!( + db.client.jobs().get(locked).await.unwrap().state, + JobState::Completed + ); + assert_eq!( + db.client.jobs().get(running).await.unwrap().state, + JobState::Running + ); + drop(connection); + lock.rollback().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn committed_transactions_preserve_savepoints_and_reuse_connections() { + let db = TestDatabase::new().await; + let pool = MySqlPoolOptions::new() + .max_connections(1) + .connect_with((*db.pool.connect_options()).clone()) + .await + .unwrap(); + let mut transaction = Transaction::begin(&pool).await.unwrap(); + let connection_id: u64 = sqlx::query_scalar("SELECT CONNECTION_ID()") + .fetch_one(&mut *transaction) + .await + .unwrap(); + let mut savepoint = sqlx::Connection::begin(&mut *transaction).await.unwrap(); + let inserted = db + .client + .insert(Args { value: 1 }) + .tx(&mut savepoint) + .await + .unwrap(); + savepoint.rollback().await.unwrap(); + transaction.commit().await.unwrap(); + + let client = Client::builder(pool.clone()).build().unwrap(); + assert!( + client + .jobs() + .list(JobListParams::default()) + .await + .unwrap() + .jobs + .is_empty() + ); + assert!(client.jobs().get(inserted.job.row.id).await.is_err()); + assert_eq!( + client + .queues() + .list(crate::QueueListParams::default()) + .await + .unwrap(), + [] as [crate::Queue; 0] + ); + + let reused_connection_id: u64 = sqlx::query_scalar("SELECT CONNECTION_ID()") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(connection_id, reused_connection_id); + pool.close().await; + db.close().await; +} + +#[tokio::test] +async fn migrations_are_serialized_and_reversible() { + let db = TestDatabase::new().await; + let migrator = MySqlMigrator::new(db.pool.clone()); + assert!(migrator.validate(None).await.unwrap().is_valid()); + migrator + .migrate(Direction::Down, MigrateOpts::new().with_target_version(-1)) + .await + .unwrap(); + assert_eq!(migrator.existing_versions().await.unwrap(), [] as [i64; 0]); + let (left, right) = tokio::join!(migrator.migrate_up(), migrator.migrate_up()); + assert_eq!(left.unwrap().len() + right.unwrap().len(), 8); + assert_eq!( + migrator.existing_versions().await.unwrap(), + (1..=8).collect::>() + ); + db.close().await; +} + +#[tokio::test] +async fn insertion_and_completion_use_the_callers_transaction() { + let db = TestDatabase::new().await; + let now = DateTime::parse_from_rfc3339("2026-01-02T03:04:05.123456Z") + .unwrap() + .with_timezone(&Utc); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let inserted = db + .client + .insert(Args { + value: 9_007_199_254_740_993, + }) + .opts( + InsertOpts::default() + .with_max_attempts(40_000) + .with_scheduled_at(now), + ) + .tx(&mut tx) + .await + .unwrap(); + assert_eq!(inserted.job.args.value, 9_007_199_254_740_993); + assert_eq!(inserted.job.row.scheduled_at, now); + assert_eq!(inserted.job.row.max_attempts, 40_000); + tx.rollback().await.unwrap(); + assert!(db.client.jobs().get(inserted.job.row.id).await.is_err()); + + let inserted = db.client.insert(Args { value: 1 }).await.unwrap(); + let id = inserted.job.row.id; + sqlx::query("UPDATE river_job SET state = 'running', attempt = 1, attempted_at = UTC_TIMESTAMP(6) WHERE id = ?").bind(id).execute(&db.pool).await.unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + db.client.jobs().complete(id).tx(&mut tx).await.unwrap(); + tx.rollback().await.unwrap(); + assert_eq!( + db.client.jobs().get(id).await.unwrap().state, + JobState::Running + ); + db.close().await; +} + +#[tokio::test] +async fn concurrent_unique_insertion_returns_one_job() { + let db = TestDatabase::new().await; + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..12 { + let client = db.client.clone(); + tasks.spawn(async move { + client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_unique(UniqueOpts::new().with_by_args(true))) + .await + .unwrap() + }); + } + let mut ids = HashSet::new(); + let mut inserted = 0; + while let Some(result) = tasks.join_next().await { + let result = result.unwrap(); + ids.insert(result.job.row.id); + inserted += usize::from(!result.unique_skipped_as_duplicate); + } + assert_eq!(ids.len(), 1); + assert_eq!(inserted, 1); + db.close().await; +} + +#[tokio::test] +async fn concurrent_claims_do_not_duplicate_jobs_and_respect_paused_queues() { + let db = TestDatabase::new().await; + for value in 0..40 { + db.client.insert(Args { value }).await.unwrap(); + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + queue_upsert(&mut tx, "paused", &Map::new(), Some(Utc::now()), Utc::now()) + .await + .unwrap(); + tx.commit().await.unwrap(); + db.client + .insert(Args { value: 0 }) + .opts(InsertOpts::default().with_queue("paused")) + .await + .unwrap(); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let pool = db.pool.clone(); + tasks.spawn(async move { + let mut tx = crate::database::begin_mysql_write(&pool).await.unwrap(); + let rows = claim( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 10, + max_attempted_by: 100, + now: Utc::now(), + queue: "default", + }, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + rows.into_iter() + .map(|row| row.unwrap().id) + .collect::>() + }); + } + let mut ids = HashSet::new(); + while let Some(result) = tasks.join_next().await { + for id in result.unwrap() { + assert!(ids.insert(id)); + } + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let params = ClaimJobs { + client_id: "worker", + kinds: None, + limit: 50, + max_attempted_by: 100, + now: Utc::now(), + queue: "default", + }; + for row in claim(&mut tx, ¶ms).await.unwrap() { + assert!(ids.insert(row.unwrap().id)); + } + assert_eq!(ids.len(), 40); + assert!( + claim( + &mut tx, + &ClaimJobs { + queue: "paused", + ..params + } + ) + .await + .unwrap() + .is_empty() + ); + tx.commit().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn cancellation_wins_over_retry_and_metadata_preserves_nulls() { + let db = TestDatabase::new().await; + let inserted = db.client.insert(Args { value: 1 }).await.unwrap(); + let id = inserted.job.row.id; + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + claim_selected( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 1, + max_attempted_by: 100, + now: Utc::now(), + queue: "default", + }, + &[id], + ) + .await + .unwrap(); + cancel(&mut tx, id, Utc::now()).await.unwrap(); + let metadata = Map::from_iter([("output".into(), Value::Null)]); + let row = complete( + &mut tx, + &CompleteJob { + attempt: Some(0), + error: None, + finalized_at: None, + id, + metadata_updates: Some(&metadata), + now: Utc::now(), + scheduled_at: Some(Utc::now()), + state: JobState::Available, + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(row.state, JobState::Cancelled); + assert_eq!(row.attempt, 1); + assert_eq!( + row.metadata.get::("output").unwrap(), + Some(Value::Null) + ); + assert!(row.finalized_at.is_some()); + tx.commit().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn metadata_containment_preserves_structure_and_exact_queue_names() { + let db = TestDatabase::new().await; + let wanted = db + .client + .insert(Args { value: 1 }) + .opts( + InsertOpts::default().with_metadata( + json!({"obj":{"a":1,"b":2},"array":[[1,2]],"nil":null}) + .as_object() + .unwrap() + .clone(), + ), + ) + .await + .unwrap(); + let other = db.client.insert(Args { value: 2 }).await.unwrap(); + sqlx::query("UPDATE river_job SET queue = 'default ' WHERE id = ?") + .bind(other.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + for (fragment, matches) in [ + (json!({"obj":{"a":1}}), true), + (json!({"obj":{"a":2}}), false), + (json!({"array":[[1]]}), true), + (json!({"array":[1]}), false), + (json!({"nil":null}), true), + (json!({"absent":null}), false), + ] { + let result = db + .client + .jobs() + .list(JobListParams::default().metadata(fragment.as_object().unwrap().clone())) + .await + .unwrap(); + assert_eq!( + result.jobs.iter().map(|job| job.id).collect::>(), + if matches { + vec![wanted.job.row.id] + } else { + vec![] + }, + "{fragment}" + ); + } + assert_eq!( + db.client + .jobs() + .list(JobListParams::default().queues(["default"])) + .await + .unwrap() + .jobs + .len(), + 1 + ); + db.close().await; +} + +#[tokio::test] +async fn outbox_waits_for_lower_uncommitted_ids() { + let db = TestDatabase::new().await; + let mut listener = NotificationListener::default(); + let mut connection = db.pool.acquire().await.unwrap(); + listener.connect(&mut connection).await.unwrap(); + listener + .listen(&mut connection, "river_leadership") + .await + .unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + notification_insert( + &mut tx, + &[NotificationInput { + topic: "river_leadership", + payload: "first", + }], + ) + .await + .unwrap(); + db.client.request_resign().await.unwrap(); + let read = tokio::spawn(async move { + let first = listener.next(&mut connection).await.unwrap().unwrap(); + let second = listener.next(&mut connection).await.unwrap().unwrap(); + (first, second) + }); + // Wait for the actual blocked scan, then verify it holds its locks in + // an explicit transaction instead of relying on autocommit semantics. + let autocommit = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let autocommit: Option = sqlx::query_scalar( + "SELECT t.AUTOCOMMIT FROM performance_schema.data_lock_waits w \ + JOIN performance_schema.data_locks l \ + ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE \ + JOIN performance_schema.events_transactions_current t \ + ON t.THREAD_ID=w.REQUESTING_THREAD_ID \ + WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'", + ) + .bind(&db.name) + .fetch_optional(&db.admin) + .await + .unwrap(); + if let Some(autocommit) = autocommit { + break autocommit; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!(autocommit, "NO"); + tx.commit().await.unwrap(); + let (first, second) = tokio::time::timeout(Duration::from_secs(5), read) + .await + .unwrap() + .unwrap(); + assert_eq!(first.payload, "first"); + assert_eq!( + serde_json::from_str::(&second.payload).unwrap()["action"], + "request_resign" + ); + assert!(first.id < second.id); + db.close().await; +} + +#[tokio::test] +async fn maintenance_schedules_rescues_and_cleans_jobs() { + let db = TestDatabase::new().await; + let inserted = db + .client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_scheduled_at(Utc::now() + chrono::Duration::hours(1))) + .await + .unwrap(); + let now = Utc::now() + chrono::Duration::hours(2); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let leader = leader_elect(&mut tx, "leader", now, Duration::from_secs(30)) + .await + .unwrap() + .unwrap(); + assert!( + leader_elect(&mut tx, "other", now, Duration::from_secs(30)) + .await + .unwrap() + .is_none() + ); + assert!( + leader_reelect( + &mut tx, + "leader", + leader.elected_at, + now, + Duration::from_mins(1) + ) + .await + .unwrap() + .is_some() + ); + assert!( + !leader_resign(&mut tx, "other", leader.elected_at) + .await + .unwrap() + ); + assert!( + leader_resign(&mut tx, "leader", leader.elected_at) + .await + .unwrap() + ); + let candidates = schedule_candidates(&mut tx, now, 10).await.unwrap(); + assert_eq!(candidates.len(), 1); + schedule_set_available(&mut tx, &[candidates[0].id]) + .await + .unwrap(); + claim( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 10, + max_attempted_by: 100, + now, + queue: "default", + }, + ) + .await + .unwrap(); + let rescue_time = now + chrono::Duration::hours(1); + let error = AttemptError { + at: rescue_time, + attempt: 1, + error: "rescued".into(), + trace: String::new(), + }; + let rescued = rescue( + &mut tx, + &RescueJob { + error: &error, + finalized_at: Some(rescue_time), + id: inserted.job.row.id, + scheduled_at: now, + state: JobState::Discarded, + stuck_horizon: rescue_time, + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(rescued.state, JobState::Discarded); + assert_eq!(rescued.errors.len(), 1); + assert_eq!( + cleanup_jobs( + &mut tx, + &CleanupJobs { + cancelled_before: None, + completed_before: None, + discarded_before: Some(rescue_time + chrono::Duration::seconds(1)), + limit: 10, + queues_excluded: &[], + queues_included: None + } + ) + .await + .unwrap(), + 1 + ); + tx.commit().await.unwrap(); + db.close().await; +} + +async fn wait_state(client: &Client, id: i64, state: JobState) -> JobRow { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let row = client.jobs().get(id).await.unwrap(); + if row.state == state { + return row; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("job {id} did not reach {state:?}")) +} + +#[tokio::test] +async fn workers_receive_peer_notifications_and_run_maintenance() { + use crate::{Job, MaintenanceConfig, QueueConfig, WorkContext, WorkOutcome, Workers}; + let db = TestDatabase::new().await; + // Use a fresh peer for each notification so insert cooldowns cannot suppress it. + // Make the job due before commit so client/server clock skew cannot consume + // the notification before the job is eligible for a claim. + let insert_available = async |value| { + let peer = Client::builder(db.pool.clone()).build().unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let inserted = peer.insert(Args { value }).tx(&mut tx).await.unwrap(); + sqlx::query("UPDATE river_job SET scheduled_at = ? WHERE id = ?") + .bind(mysql_time(Utc::now() - chrono::Duration::seconds(1))) + .bind(inserted.job.row.id) + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + inserted + }; + let mut workers = Workers::new(); + workers + .add_fn(|context: WorkContext, job: Job| async move { + if job.args.value < 0 { + context.cancellation_token().cancelled().await; + return Ok::<_, crate::BoxError>(WorkOutcome::Cancel); + } + context.record_output(json!({"value": job.args.value}))?; + Ok(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .id("mysql-worker") + .workers(workers) + .queue( + "default", + QueueConfig::new(4) + .with_fetch_cooldown(Duration::from_millis(1)) + .with_fetch_poll_interval(Duration::from_mins(1)), + ) + .maintenance( + MaintenanceConfig::default() + .with_elect_interval(Duration::from_millis(50)) + .with_scheduler_interval(Duration::from_millis(50)), + ) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let inserted = insert_available(42).await; + let completed = wait_state(&db.client, inserted.job.row.id, JobState::Completed).await; + assert_eq!( + completed.metadata.get::("output").unwrap(), + Some(json!({"value":42})) + ); + + let cancellation = insert_available(-1).await; + wait_state(&db.client, cancellation.job.row.id, JobState::Running).await; + db.client + .jobs() + .cancel(cancellation.job.row.id) + .await + .unwrap(); + wait_state(&db.client, cancellation.job.row.id, JobState::Cancelled).await; + + let scheduled = db + .client + .insert(Args { value: 5 }) + .opts(InsertOpts::default().with_scheduled_at(Utc::now() + chrono::Duration::hours(1))) + .await + .unwrap(); + sqlx::query( + "UPDATE river_job SET scheduled_at = UTC_TIMESTAMP(6) - INTERVAL 1 SECOND WHERE id = ?", + ) + .bind(scheduled.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + wait_state(&db.client, scheduled.job.row.id, JobState::Completed).await; + + let term: DateTime = + sqlx::query_scalar("SELECT elected_at FROM river_leader WHERE name = 'default'") + .fetch_one(&db.pool) + .await + .unwrap(); + db.client.request_resign().await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let current: Option> = + sqlx::query_scalar("SELECT elected_at FROM river_leader WHERE name = 'default'") + .fetch_optional(&db.pool) + .await + .unwrap(); + if current != Some(term) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn shutdown_interrupts_a_blocked_notification_poll() { + use crate::{Job, QueueConfig, WorkContext, WorkOutcome, Workers}; + let db = TestDatabase::new().await; + let mut workers = Workers::new(); + workers + .add_fn(|_: WorkContext, _: Job| async { + Ok::<_, crate::BoxError>(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let mut tx = db.pool.begin().await.unwrap(); + sqlx::query("INSERT INTO river_notification (topic, payload) VALUES ('river_insert', '{}')") + .execute(&mut *tx) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'") + .bind(&db.name).fetch_one(&db.admin).await.unwrap(); + if waiting > 0 { break; } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), run.stop()) + .await + .unwrap() + .unwrap(); + tx.rollback().await.unwrap(); + db.close().await; +} diff --git a/rust/riverqueue/src/database/mysql/tests/conformance.rs b/rust/riverqueue/src/database/mysql/tests/conformance.rs new file mode 100644 index 000000000..f4f1ec20e --- /dev/null +++ b/rust/riverqueue/src/database/mysql/tests/conformance.rs @@ -0,0 +1,348 @@ +//! Persist Go-generated protocol values through the real MySQL backend. + +use super::*; +use crate::{Job, QueueConfig, QueueUpdateParams, WorkContext, WorkOutcome, Workers}; + +#[derive(Deserialize)] +struct NotificationFixture { + name: String, + payload: Value, + topic: String, +} + +#[derive(Deserialize)] +struct Protocol { + attempt_error: AttemptError, + job_states: Vec, + metadata_keys: HashMap, + notifications: Vec, +} + +#[derive(Deserialize)] +struct SnoozeFixture { + expected_snoozes: i64, + metadata: JobMetadata, + name: String, +} + +#[derive(Deserialize)] +struct Snoozes { + snooze_counters: Vec, +} + +#[derive(Deserialize)] +struct StateFixture { + state: JobState, + unique_bit: u8, +} + +fn protocol() -> Protocol { + serde_json::from_str(&crate::conformance::read_fixture("protocol_values.json")).unwrap() +} + +#[tokio::test] +async fn emits_go_notification_payloads() { + let db = TestDatabase::new().await; + let mut connection = db.pool.acquire().await.unwrap(); + queue_upsert(&mut connection, "priority", &Map::new(), None, Utc::now()) + .await + .unwrap(); + + for fixture in protocol().notifications { + match fixture.name.as_str() { + "cancel" => { + let job = db + .client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_queue("priority")) + .await + .unwrap(); + sqlx::query("UPDATE river_job SET id=42, state='running' WHERE id=?") + .bind(job.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + db.client.jobs().cancel(42).await.unwrap(); + } + "insert" => { + // A fresh client has no local insert-notification cooldown. + Client::builder(db.pool.clone()) + .build() + .unwrap() + .insert(Args { value: 2 }) + .opts(InsertOpts::default().with_queue("priority")) + .await + .unwrap(); + } + "metadata_changed" => { + db.client + .queues() + .update( + "priority", + QueueUpdateParams::new() + .metadata(Map::from_iter([("owner".into(), json!("candidate"))])), + ) + .await + .unwrap(); + } + "pause" => db.client.queues().pause("priority").await.unwrap(), + "request_resign" => db.client.request_resign().await.unwrap(), + "resigned" => { + // Like Go's MySQL/SQLite drivers, Rust does not emit a + // resignation notification. Receiving it is tested below. + let before = notification_last_id(&mut connection).await.unwrap(); + let leader = leader_elect( + &mut connection, + "client-1", + Utc::now(), + Duration::from_mins(1), + ) + .await + .unwrap() + .unwrap(); + assert!( + leader_resign(&mut connection, "client-1", leader.elected_at) + .await + .unwrap() + ); + assert_eq!(notification_last_id(&mut connection).await.unwrap(), before); + continue; + } + "resume" => db.client.queues().resume("priority").await.unwrap(), + name => panic!("unhandled Go notification {name}"), + } + let (topic, payload): (String, String) = sqlx::query_as( + "SELECT topic, CAST(payload AS CHAR) FROM river_notification ORDER BY id DESC LIMIT 1", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(topic, fixture.topic, "{}", fixture.name); + assert_eq!( + serde_json::from_str::(&payload).unwrap(), + fixture.payload, + "{}", + fixture.name + ); + } + drop(connection); + db.close().await; +} + +#[tokio::test] +async fn persists_go_snooze_counters_through_workers() { + let db = TestDatabase::new().await; + let fixture: Snoozes = + serde_json::from_str(&crate::conformance::read_fixture("snooze_counters.json")).unwrap(); + assert!(!fixture.snooze_counters.is_empty()); + let mut workers = Workers::new(); + workers + .add_fn(|_: WorkContext, _: Job| async { + Ok::<_, crate::BoxError>(WorkOutcome::Snooze(Duration::from_hours(1))) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + for case in fixture.snooze_counters { + let inserted = db + .client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_metadata(case.metadata)) + .await + .unwrap(); + let stored = wait_state(&db.client, inserted.job.row.id, JobState::Scheduled).await; + assert_eq!(stored.attempt, 0, "{}", case.name); + assert_eq!( + stored.metadata.get::("snoozes").unwrap(), + Some(case.expected_snoozes), + "{}", + case.name + ); + } + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn receives_go_notifications_after_commit_and_ignores_rollback() { + let db = TestDatabase::new().await; + let mut connection = db.pool.acquire().await.unwrap(); + let mut listener = NotificationListener::default(); + listener.connect(&mut connection).await.unwrap(); + let fixture = protocol(); + for notification in &fixture.notifications { + listener + .listen(&mut connection, ¬ification.topic) + .await + .unwrap(); + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + for notification in &fixture.notifications { + sqlx::query("INSERT INTO river_notification(topic,payload) VALUES (?,?)") + .bind(¬ification.topic) + .bind(notification.payload.to_string()) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.rollback().await.unwrap(); + assert!(listener.next(&mut connection).await.unwrap().is_none()); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + for notification in &fixture.notifications { + sqlx::query("INSERT INTO river_notification(topic,payload) VALUES (?,?)") + .bind(¬ification.topic) + .bind(notification.payload.to_string()) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + for expected in fixture.notifications { + let received = listener.next(&mut connection).await.unwrap().unwrap(); + assert_eq!(received.topic, expected.topic); + assert_eq!( + serde_json::from_str::(&received.payload).unwrap(), + expected.payload + ); + } + assert!(listener.next(&mut connection).await.unwrap().is_none()); + drop(connection); + db.close().await; +} + +#[tokio::test] +async fn round_trips_go_errors_and_persists_rescue_and_output_metadata() { + let db = TestDatabase::new().await; + let fixture = protocol(); + let keys = &fixture.metadata_keys; + let inserted = db + .client + .insert(Args { value: 1 }) + .opts( + InsertOpts::default() + .with_metadata(Map::from_iter([(keys["rescue_count"].clone(), json!(2))])), + ) + .await + .unwrap(); + let id = inserted.job.row.id; + let nonce = inserted + .job + .row + .metadata + .get::(&keys["unique_nonce"]) + .unwrap() + .unwrap(); + assert_ne!(nonce, ""); + sqlx::query("UPDATE river_job SET state='running', attempt=3, attempted_at=UTC_TIMESTAMP(6), errors=JSON_ARRAY(CAST(? AS JSON)) WHERE id=?") + .bind(serde_json::to_string(&fixture.attempt_error).unwrap()).bind(id).execute(&db.pool).await.unwrap(); + assert_eq!( + db.client.jobs().get(id).await.unwrap().errors, + vec![fixture.attempt_error.clone()] + ); + let now = Utc::now(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let rescued = rescue( + &mut tx, + &RescueJob { + error: &fixture.attempt_error, + finalized_at: None, + id, + scheduled_at: now, + state: JobState::Retryable, + stuck_horizon: now + chrono::Duration::hours(1), + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + // MySQL JSON arithmetic returns a double, as in Go's driver. + rescued.metadata.get::(&keys["rescue_count"]).unwrap(), + Some(3.0) + ); + tx.commit().await.unwrap(); + let mut workers = Workers::new(); + workers + .add_fn(|context: WorkContext, _: Job| async move { + context.record_output(json!({"ok": true}))?; + Ok::<_, crate::BoxError>(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + db.client.jobs().retry(id).await.unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let stored = wait_state(&db.client, id, JobState::Completed).await; + assert_eq!( + stored.metadata.get::(&keys["output"]).unwrap(), + Some(json!({"ok": true})) + ); + assert_eq!( + stored.metadata.get::(&keys["rescue_count"]).unwrap(), + Some(3.0) + ); + assert_eq!( + stored + .metadata + .get::(&keys["unique_nonce"]) + .unwrap(), + Some(nonce) + ); + assert_eq!( + stored.errors, + vec![fixture.attempt_error.clone(), fixture.attempt_error] + ); + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn stored_states_and_unique_bits_match_go() { + let db = TestDatabase::new().await; + let opts = InsertOpts::default().with_unique( + UniqueOpts::new() + .with_by_args(true) + .with_by_state(JobState::ALL.iter().copied()), + ); + let inserted = db + .client + .insert(Args { value: 1 }) + .opts(opts.clone()) + .await + .unwrap(); + for fixture in protocol().job_states { + let finalized = matches!( + fixture.state, + JobState::Cancelled | JobState::Completed | JobState::Discarded + ); + sqlx::query("UPDATE river_job SET state=?, unique_states=?, finalized_at=IF(?, UTC_TIMESTAMP(6), NULL) WHERE id=?") + .bind(fixture.state.as_str()).bind(fixture.unique_bit).bind(finalized).bind(inserted.job.row.id).execute(&db.pool).await.unwrap(); + let stored = db.client.jobs().get(inserted.job.row.id).await.unwrap(); + assert_eq!(stored.state, fixture.state); + assert_eq!(stored.unique_states, Some(vec![fixture.state])); + let duplicate = db + .client + .insert(Args { value: 1 }) + .opts(opts.clone()) + .await + .unwrap(); + assert!(duplicate.unique_skipped_as_duplicate, "{}", fixture.state); + assert_eq!(duplicate.job.row.id, inserted.job.row.id); + let bit: i16 = sqlx::query_scalar("SELECT unique_states FROM river_job WHERE id=?") + .bind(inserted.job.row.id) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(bit, i16::from(fixture.unique_bit)); + } + db.close().await; +} diff --git a/rust/riverqueue/src/lib.rs b/rust/riverqueue/src/lib.rs index 058cecae7..9ad5aae20 100644 --- a/rust/riverqueue/src/lib.rs +++ b/rust/riverqueue/src/lib.rs @@ -1,4 +1,4 @@ -// The README's examples use both backends, so it's the crate documentation +// The README's Postgres and SQLite examples need both features, so it's the crate documentation // (and its examples run as doctests) when both are enabled, as on docs.rs. #![cfg_attr( all(feature = "postgres", feature = "sqlite"), @@ -14,8 +14,10 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -#[cfg(not(any(feature = "postgres", feature = "sqlite")))] -compile_error!("riverqueue requires at least one database feature: `postgres` or `sqlite`"); +#[cfg(not(any(feature = "postgres", feature = "sqlite", feature = "mysql")))] +compile_error!( + "riverqueue requires at least one database feature: `mysql`, `postgres`, or `sqlite`" +); extern crate self as riverqueue; diff --git a/rust/riverqueue/src/maintenance/cleaner.rs b/rust/riverqueue/src/maintenance/cleaner.rs index 63fd25679..c5984c36f 100644 --- a/rust/riverqueue/src/maintenance/cleaner.rs +++ b/rust/riverqueue/src/maintenance/cleaner.rs @@ -1,4 +1,4 @@ -//! Deletes expired finalized jobs, idle queues, and (on SQLite) delivered +//! Deletes expired finalized jobs, idle queues, and (on MySQL/SQLite) delivered //! notifications, ports of Go's `JobCleaner`, `QueueCleaner`, and //! `SQLiteNotificationCleaner`. @@ -15,6 +15,8 @@ use crate::__private::{ use crate::Error; use crate::client::ClientInner; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::pilot::job_delete_error; @@ -25,11 +27,11 @@ use super::{ }; /// Interval of the SQLite notification cleaner. -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] pub(super) const NOTIFICATION_CLEANER_INTERVAL: Duration = Duration::from_mins(1); /// Age after which SQLite notification outbox rows are deleted. -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] const NOTIFICATION_RETENTION: Duration = Duration::from_mins(5); fn horizon( @@ -81,6 +83,21 @@ async fn clean_jobs_batch( ) -> Result { let timeout = context.inner.maintenance.job_cleaner_timeout; match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let operation = async { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let count = job_delete_before( + &context.inner, + DatabaseConnection::MySql(&mut transaction), + params, + ) + .await?; + transaction.commit().await?; + Ok::<_, MaintenanceError>(count) + }; + super::mysql_cancellable(&context.cancel, timeout, operation).await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let operation = async { @@ -168,6 +185,10 @@ pub(crate) async fn standard_job_delete_before( DatabaseConnection::Postgres(connection) => { Ok(postgres_delete_finalized_jobs(connection, &inner.schema, params).await?) } + #[cfg(feature = "mysql")] + DatabaseConnection::MySql(connection) => mysql_delete_finalized_jobs(connection, params) + .await + .map_err(|error| Error::Database(error.into())), #[cfg(feature = "sqlite")] DatabaseConnection::Sqlite(connection) => sqlite_delete_finalized_jobs(connection, params) .await @@ -216,6 +237,34 @@ pub(crate) async fn postgres_delete_finalized_jobs( Ok(result.rows_affected()) } +/// Runs the job cleaner's deletion on MySQL. +#[cfg(feature = "mysql")] +pub(crate) async fn mysql_delete_finalized_jobs( + connection: &mut sqlx::MySqlConnection, + params: &FinalizedJobDeleteParams, +) -> Result { + let queues_excluded = params + .queues_excluded + .iter() + .map(String::as_str) + .collect::>(); + let queues_included = params + .queues_included + .as_ref() + .map(|queues| queues.iter().map(String::as_str).collect::>()); + mysql::cleanup_jobs( + connection, + &mysql::CleanupJobs { + cancelled_before: params.cancelled_before, + completed_before: params.completed_before, + discarded_before: params.discarded_before, + limit: i32::try_from(params.limit).unwrap_or(i32::MAX), + queues_excluded: &queues_excluded, + queues_included: queues_included.as_deref(), + }, + ) + .await +} /// Runs the job cleaner's deletion on SQLite. #[cfg(feature = "sqlite")] pub(crate) async fn sqlite_delete_finalized_jobs( @@ -268,6 +317,21 @@ async fn clean_queues_batch( limit: i64, ) -> Result { match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let operation = async { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let deleted = mysql::queue_delete_expired( + &mut transaction, + updated_before, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await?; + transaction.commit().await?; + Ok::<_, MaintenanceError>(deleted.len()) + }; + super::mysql_cancellable(&context.cancel, TIMEOUT_DEFAULT, operation).await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let operation = async { @@ -312,6 +376,40 @@ async fn clean_queues_batch( } } +/// Deletes MySQL notification outbox rows old enough that every poller has +/// consumed them, oldest first, in batches like River Go's +/// `MySQLNotificationCleaner`. +/// +/// Each batch commits on its own, releasing its row locks before the next batch. The horizon is fixed for the pass +/// so rows expiring meanwhile don't extend it. Repeated timeouts switch to +/// the reduced batch size. +#[cfg(feature = "mysql")] +pub(super) async fn clean_mysql_notifications( + context: &ServiceContext, + pool: &sqlx::MySqlPool, +) -> Result { + let retention = chrono::Duration::from_std(NOTIFICATION_RETENTION) + .map_err(|error| Error::configuration_context("maintenance", error.to_string()))?; + let created_before = Utc::now() - retention; + let mut deleted = 0; + loop { + let limit = batch_size(&context.breakers.notification_cleaner); + let operation = async { + let mut connection = mysql::Connection::acquire(pool).await?; + let count = mysql::notification_cleanup(&mut connection, created_before, limit).await?; + connection.release(); + Ok::<_, MaintenanceError>(count) + }; + let result = super::mysql_cancellable(&context.cancel, TIMEOUT_DEFAULT, operation).await; + record_batch(&context.breakers.notification_cleaner, &result); + let count = result?; + deleted += count; + if count < u64::try_from(limit).unwrap_or(u64::MAX) { + return Ok(deleted); + } + batch_backoff(&context.cancel).await?; + } +} /// Deletes SQLite notification outbox rows old enough that every poller has /// consumed them, oldest first, in batches like River Go's /// `SQLiteNotificationCleaner`. diff --git a/rust/riverqueue/src/maintenance/elector.rs b/rust/riverqueue/src/maintenance/elector.rs index c3905c8b6..523d81913 100644 --- a/rust/riverqueue/src/maintenance/elector.rs +++ b/rust/riverqueue/src/maintenance/elector.rs @@ -11,6 +11,8 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::{Error, client::ClientInner}; @@ -445,6 +447,19 @@ impl DatabaseLeaderStore { impl LeaderStore for DatabaseLeaderStore { async fn elect(&self, ttl: Duration) -> Result>, Error> { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let now = Utc::now(); + mysql::leader_delete_expired(&mut transaction, now) + .await + .map_err(mysql_error)?; + let leader = mysql::leader_elect(&mut transaction, &self.inner.id, now, ttl) + .await + .map_err(mysql_error)?; + transaction.commit().await?; + Ok(leader.map(|leader| leader.elected_at)) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -488,6 +503,21 @@ impl LeaderStore for DatabaseLeaderStore { ttl: Duration, ) -> Result>, Error> { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let leader = mysql::leader_reelect( + &mut transaction, + &self.inner.id, + elected_at, + Utc::now(), + ttl, + ) + .await + .map_err(mysql_error)?; + transaction.commit().await?; + Ok(leader.map(|leader| leader.elected_at)) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -524,6 +554,17 @@ impl LeaderStore for DatabaseLeaderStore { async fn resign(&self, elected_at: DateTime) -> Result { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let resigned = mysql::leader_resign(&mut transaction, &self.inner.id, elected_at) + .await + .map_err(mysql_error)?; + // Like Go's MySQL driver, a resignation writes no outbox + // row; other clients bid at their next election attempt. + transaction.commit().await?; + Ok(resigned) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -567,6 +608,10 @@ impl LeaderStore for DatabaseLeaderStore { } } +#[cfg(feature = "mysql")] +fn mysql_error(error: mysql::BackendError) -> Error { + Error::Database(error.into()) +} #[cfg(feature = "sqlite")] fn sqlite_error(error: sqlite::BackendError) -> Error { Error::Database(error.into()) diff --git a/rust/riverqueue/src/maintenance/maintainer.rs b/rust/riverqueue/src/maintenance/maintainer.rs index c707c101d..8f8156460 100644 --- a/rust/riverqueue/src/maintenance/maintainer.rs +++ b/rust/riverqueue/src/maintenance/maintainer.rs @@ -144,6 +144,23 @@ async fn run_term( |context| Box::pin(async move { cleaner::clean_queues(&context).await }), )); match inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let pool = pool.clone(); + services.spawn(run_periodically( + Arc::clone(&context), + "MySQL notification cleaner", + cleaner::NOTIFICATION_CLEANER_INTERVAL, + move |context| { + let pool = pool.clone(); + Box::pin(async move { + cleaner::clean_mysql_notifications(&context, &pool) + .await + .map(|_| ()) + }) + }, + )); + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let pool = pool.clone(); diff --git a/rust/riverqueue/src/maintenance/mod.rs b/rust/riverqueue/src/maintenance/mod.rs index 1f674b4bf..ab747ab37 100644 --- a/rust/riverqueue/src/maintenance/mod.rs +++ b/rust/riverqueue/src/maintenance/mod.rs @@ -29,6 +29,8 @@ mod scheduler; #[cfg(all(test, feature = "postgres", river_postgres_tests))] mod tests; +#[cfg(feature = "mysql")] +pub(crate) use cleaner::mysql_delete_finalized_jobs; #[cfg(feature = "postgres")] pub(crate) use cleaner::postgres_delete_finalized_jobs; #[cfg(feature = "sqlite")] @@ -190,6 +192,12 @@ impl From for MaintenanceError { } } +#[cfg(feature = "mysql")] +impl From for MaintenanceError { + fn from(error: crate::database::mysql::BackendError) -> Self { + Self::Failed(Error::Database(error.into())) + } +} #[cfg(feature = "sqlite")] impl From for MaintenanceError { fn from(error: crate::database::sqlite::BackendError) -> Self { @@ -267,7 +275,7 @@ impl ReducedBatchBreaker { #[derive(Debug)] pub(crate) struct Breakers { pub(crate) job_cleaner: Mutex, - #[cfg(feature = "sqlite")] + #[cfg(any(feature = "sqlite", feature = "mysql"))] pub(crate) notification_cleaner: Mutex, pub(crate) queue_cleaner: Mutex, pub(crate) rescuer: Mutex, @@ -278,7 +286,7 @@ impl Breakers { fn new(sizes: BatchSizes) -> Self { Self { job_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), - #[cfg(feature = "sqlite")] + #[cfg(any(feature = "sqlite", feature = "mysql"))] notification_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), queue_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), rescuer: Mutex::new(ReducedBatchBreaker::new(sizes)), @@ -361,6 +369,28 @@ async fn batch_backoff(cancel: &CancellationToken) -> Result<(), MaintenanceErro } } +/// Runs a MySQL operation under the term token and a client-side deadline. +/// +/// Dropping a batch also drops its transaction; SQLx rolls it back before +/// reusing the connection. +#[cfg(feature = "mysql")] +async fn mysql_cancellable( + cancel: &CancellationToken, + timeout: Duration, + operation: impl Future>, +) -> Result +where + MaintenanceError: From, +{ + tokio::select! { + biased; + () = cancel.cancelled() => Err(MaintenanceError::Cancelled), + result = tokio::time::timeout(timeout, operation) => match result { + Ok(result) => result.map_err(MaintenanceError::from), + Err(_) => Err(MaintenanceError::TimedOut), + }, + } +} /// Runs a SQLite operation under the term token and a client-side deadline. /// /// SQLite executes statements on the client, so dropping the future cannot diff --git a/rust/riverqueue/src/maintenance/rescuer.rs b/rust/riverqueue/src/maintenance/rescuer.rs index 9030f4c78..0a1578c99 100644 --- a/rust/riverqueue/src/maintenance/rescuer.rs +++ b/rust/riverqueue/src/maintenance/rescuer.rs @@ -13,6 +13,8 @@ use crate::__private::{ #[cfg(feature = "postgres")] use crate::client::{JobRecord, job_projection, tolerant_row}; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::{AttemptError, Error, JobRow, JobState, WorkerTimeout, client::ClientInner}; @@ -64,6 +66,15 @@ async fn rescue_batch( stuck_horizon: DateTime, ) -> Result { match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + super::mysql_cancellable( + &context.cancel, + TIMEOUT_DEFAULT, + rescue_batch_mysql(&context.inner, pool, after_id, limit, stuck_horizon), + ) + .await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { super::sqlite_cancellable( @@ -386,6 +397,92 @@ async fn rescue_batch_postgres( Ok(batch) } +#[cfg(feature = "mysql")] +async fn rescue_batch_mysql( + inner: &ClientInner, + pool: &sqlx::MySqlPool, + after_id: i64, + limit: i64, + stuck_horizon: DateTime, +) -> Result { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let selected_ids = if inner.pilot.intercepts_rescue() { + inner + .pilot + .select_rescue_job_ids( + PilotDatabaseConnection::MySql(&mut transaction), + &rescue_params(inner, after_id, limit, stuck_horizon), + ) + .await + .map_err(|source| Error::Extension { + phase: crate::ExtensionPhase::AddOn { + operation: "rescue selection", + }, + source, + })? + } else { + None + }; + let rows = match selected_ids { + Some(ids) => mysql::jobs_by_ids(&mut transaction, &ids).await?, + None => { + mysql::stuck_jobs( + &mut transaction, + after_id, + stuck_horizon, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await? + } + }; + let batch = Batch { + last_id: rows.last().map(|row| row.id), + selected: rows.len(), + }; + let jobs = rescue_jobs(inner, &rows, Utc::now())?; + if !jobs.is_empty() { + let params = RescueManyParams { + database: inner.pilot_database_config(), + jobs, + stuck_horizon, + }; + let action = if inner.pilot.intercepts_rescue() { + inner + .pilot + .rescue_jobs(PilotDatabaseConnection::MySql(&mut transaction), ¶ms) + .await + .map_err(|source| Error::Extension { + phase: crate::ExtensionPhase::AddOn { + operation: "rescue", + }, + source, + })? + } else { + RescueAction::Continue + }; + if action == RescueAction::Continue { + for job in ¶ms.jobs { + let state = job.state; + let error: AttemptError = + serde_json::from_value(job.attempt_error.clone()).map_err(Error::from)?; + mysql::rescue( + &mut transaction, + &mysql::RescueJob { + error: &error, + finalized_at: job.finalized_at, + id: job.id, + scheduled_at: job.scheduled_at, + state, + stuck_horizon, + }, + ) + .await?; + } + } + } + transaction.commit().await?; + Ok(batch) +} #[cfg(feature = "sqlite")] async fn rescue_batch_sqlite( inner: &ClientInner, diff --git a/rust/riverqueue/src/maintenance/scheduler.rs b/rust/riverqueue/src/maintenance/scheduler.rs index 03dad9571..9b69cfecd 100644 --- a/rust/riverqueue/src/maintenance/scheduler.rs +++ b/rust/riverqueue/src/maintenance/scheduler.rs @@ -7,9 +7,11 @@ use chrono::{DateTime, Utc}; #[cfg(feature = "postgres")] use sqlx::{AssertSqlSafe, Row}; -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] use crate::client::InsertNotifyLimiter; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "postgres")] use crate::database::postgres_capabilities::CapabilitiesCache; #[cfg(feature = "sqlite")] @@ -53,6 +55,20 @@ async fn schedule_batch(context: &ServiceContext, limit: i64) -> Result { + super::mysql_cancellable( + &context.cancel, + TIMEOUT_DEFAULT, + schedule_batch_mysql( + pool, + &context.inner.insert_notify_limiter, + look_ahead, + limit, + ), + ) + .await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { super::sqlite_cancellable( @@ -207,6 +223,67 @@ async fn delivers_notifications( .supports_listen_notify) } +#[cfg(feature = "mysql")] +async fn schedule_batch_mysql( + pool: &sqlx::MySqlPool, + notify_limiter: &InsertNotifyLimiter, + look_ahead: DateTime, + limit: i64, +) -> Result { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let candidates = mysql::schedule_candidates( + &mut transaction, + look_ahead, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await?; + let count = candidates.len(); + let mut available_ids = Vec::new(); + let mut conflict_ids = Vec::new(); + let mut scheduled = Vec::new(); + for candidate in candidates { + let Some(unique_key) = candidate.unique_key.as_deref() else { + available_ids.push(candidate.id); + continue; + }; + if mysql::schedule_has_unique_collision(&mut transaction, candidate.id, unique_key).await? { + conflict_ids.push(candidate.id); + } else { + // Transition unique jobs one at a time so that a later duplicate + // in the same batch observes the earlier one as a collision. + let available = + mysql::schedule_set_available(&mut transaction, &[candidate.id]).await?; + scheduled.extend(available.into_iter().map(|job| Scheduled { + queue: job.queue, + scheduled_at: job.scheduled_at, + })); + } + } + if !available_ids.is_empty() { + let available = mysql::schedule_set_available(&mut transaction, &available_ids).await?; + scheduled.extend(available.into_iter().map(|job| Scheduled { + queue: job.queue, + scheduled_at: job.scheduled_at, + })); + } + if !conflict_ids.is_empty() { + mysql::schedule_discard_conflicts(&mut transaction, &conflict_ids, look_ahead).await?; + } + let notified = notified_queues(&scheduled); + for queue in notify_limiter.due(notified.iter().map(String::as_str)) { + let payload = crate::protocol::insert_notification_payload(queue); + mysql::notification_insert( + &mut transaction, + &[mysql::NotificationInput { + payload: &payload, + topic: crate::NOTIFICATION_TOPIC_INSERT, + }], + ) + .await?; + } + transaction.commit().await?; + Ok(count) +} #[cfg(feature = "sqlite")] async fn schedule_batch_sqlite( pool: &sqlx::SqlitePool, diff --git a/rust/riverqueue/src/pilot.rs b/rust/riverqueue/src/pilot.rs index 90d8835e1..5db40e8e3 100644 --- a/rust/riverqueue/src/pilot.rs +++ b/rust/riverqueue/src/pilot.rs @@ -12,6 +12,7 @@ use serde_json::{Map, Value}; use sqlx::Postgres; #[cfg(feature = "sqlite")] use sqlx::Sqlite; +#[cfg(any(feature = "postgres", feature = "sqlite"))] use sqlx::Transaction; use tokio_util::sync::CancellationToken; @@ -52,6 +53,10 @@ impl PilotDatabase { DatabasePool::Postgres(pool) => { PilotTransactionInner::Postgres(crate::database::begin_postgres(pool).await?) } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => PilotTransactionInner::MySql( + crate::database::mysql::Transaction::begin(pool).await?, + ), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { PilotTransactionInner::Sqlite(crate::database::begin_sqlite_write(pool).await?) @@ -95,6 +100,8 @@ pub struct PilotTransaction(PilotTransactionInner); enum PilotTransactionInner { #[cfg(feature = "postgres")] Postgres(Transaction<'static, Postgres>), + #[cfg(feature = "mysql")] + MySql(crate::database::mysql::Transaction), #[cfg(feature = "sqlite")] Sqlite(Transaction<'static, Sqlite>), } @@ -109,6 +116,8 @@ impl PilotTransaction { match self.0 { #[cfg(feature = "postgres")] PilotTransactionInner::Postgres(transaction) => transaction.commit().await?, + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(transaction) => transaction.commit().await?, #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(transaction) => transaction.commit().await?, } @@ -122,6 +131,8 @@ impl PilotTransaction { PilotTransactionInner::Postgres(transaction) => { DatabaseConnection::Postgres(transaction) } + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(transaction) => DatabaseConnection::MySql(transaction), #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(transaction) => DatabaseConnection::Sqlite(transaction), } @@ -133,6 +144,8 @@ impl fmt::Debug for PilotTransaction { let kind = match &self.0 { #[cfg(feature = "postgres")] PilotTransactionInner::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(_) => DatabaseKind::Sqlite, }; diff --git a/rust/riverqueue/src/storage.rs b/rust/riverqueue/src/storage.rs index a0f0851f4..24b4be493 100644 --- a/rust/riverqueue/src/storage.rs +++ b/rust/riverqueue/src/storage.rs @@ -11,6 +11,8 @@ //! it to the connection, [`AnyBackend`], and [`Session`] enums; the //! operations themselves don't change. +#[cfg(feature = "mysql")] +mod mysql; #[cfg(feature = "postgres")] mod postgres; #[cfg(feature = "sqlite")] @@ -21,6 +23,7 @@ use serde_json::{Map, Value}; use sqlx::Postgres; #[cfg(feature = "sqlite")] use sqlx::Sqlite; +#[cfg(any(feature = "postgres", feature = "sqlite"))] use sqlx::{Transaction, pool::PoolConnection}; use crate::__private::{ @@ -121,6 +124,8 @@ pub(crate) trait Backend { enum AnyBackend<'c> { #[cfg(feature = "postgres")] Postgres(postgres::PostgresBackend<'c>), + #[cfg(feature = "mysql")] + MySql(mysql::MySqlBackend<'c>), #[cfg(feature = "sqlite")] Sqlite(sqlite::SqliteBackend<'c>), } @@ -131,6 +136,8 @@ macro_rules! dispatch { match $any { #[cfg(feature = "postgres")] AnyBackend::Postgres($backend) => $call, + #[cfg(feature = "mysql")] + AnyBackend::MySql($backend) => $call, #[cfg(feature = "sqlite")] AnyBackend::Sqlite($backend) => $call, } @@ -241,6 +248,10 @@ impl<'c> Storage<'c> { schema: &inner.schema, }) } + #[cfg(feature = "mysql")] + DatabaseConnection::MySql(connection) => { + AnyBackend::MySql(mysql::MySqlBackend { connection }) + } #[cfg(feature = "sqlite")] DatabaseConnection::Sqlite(connection) => { AnyBackend::Sqlite(sqlite::SqliteBackend { connection }) @@ -527,8 +538,12 @@ enum SessionConnection<'a> { PostgresConnection(PoolConnection), #[cfg(feature = "postgres")] PostgresTransaction(Transaction<'static, Postgres>), + #[cfg(feature = "mysql")] + MySqlConnection(crate::database::mysql::Connection), #[cfg(feature = "sqlite")] SqliteConnection(PoolConnection), + #[cfg(feature = "mysql")] + MySqlTransaction(crate::database::mysql::Transaction), #[cfg(feature = "sqlite")] SqliteTransaction(Transaction<'static, Sqlite>), } @@ -552,10 +567,20 @@ impl<'a> Session<'a> { (DatabasePool::Postgres(pool), Access::Transaction) => { SessionConnection::PostgresTransaction(crate::database::begin_postgres(pool).await?) } + #[cfg(feature = "mysql")] + (DatabasePool::MySql(pool), Access::Autocommit) => SessionConnection::MySqlConnection( + crate::database::mysql::Connection::acquire(pool).await?, + ), #[cfg(feature = "sqlite")] (DatabasePool::Sqlite(pool), Access::Autocommit) => { SessionConnection::SqliteConnection(pool.acquire().await?) } + #[cfg(feature = "mysql")] + (DatabasePool::MySql(pool), Access::Transaction) => { + SessionConnection::MySqlTransaction( + crate::database::mysql::Transaction::begin(pool).await?, + ) + } #[cfg(feature = "sqlite")] (DatabasePool::Sqlite(pool), Access::Transaction) => { SessionConnection::SqliteTransaction( @@ -578,10 +603,16 @@ impl<'a> Session<'a> { SessionConnection::PostgresTransaction(transaction) => { DatabaseConnection::Postgres(transaction) } + #[cfg(feature = "mysql")] + SessionConnection::MySqlConnection(connection) => DatabaseConnection::MySql(connection), #[cfg(feature = "sqlite")] SessionConnection::SqliteConnection(connection) => { DatabaseConnection::Sqlite(connection) } + #[cfg(feature = "mysql")] + SessionConnection::MySqlTransaction(transaction) => { + DatabaseConnection::MySql(transaction) + } #[cfg(feature = "sqlite")] SessionConnection::SqliteTransaction(transaction) => { DatabaseConnection::Sqlite(transaction) @@ -590,19 +621,23 @@ impl<'a> Session<'a> { Storage::new(inner, connection) } - /// Commits a transaction River owns. A caller-managed transaction is - /// left for the caller to commit, and an autocommit connection has - /// nothing to commit. Dropping a session instead rolls River's own - /// transaction back. + /// Finishes the operation, committing River-owned transactions and returning + /// autocommit connections to their pools. Caller-managed transactions remain + /// uncommitted. Dropping a session instead rolls back River-owned transactions + /// and discards MySQL connections to avoid reusing an interrupted operation. pub(crate) async fn commit(self) -> Result<(), Error> { match self.connection { #[cfg(feature = "postgres")] SessionConnection::PostgresTransaction(transaction) => transaction.commit().await?, + #[cfg(feature = "mysql")] + SessionConnection::MySqlTransaction(transaction) => transaction.commit().await?, #[cfg(feature = "sqlite")] SessionConnection::SqliteTransaction(transaction) => transaction.commit().await?, SessionConnection::Caller(_) => {} #[cfg(feature = "postgres")] SessionConnection::PostgresConnection(_) => {} + #[cfg(feature = "mysql")] + SessionConnection::MySqlConnection(connection) => connection.release(), #[cfg(feature = "sqlite")] SessionConnection::SqliteConnection(_) => {} } @@ -635,8 +670,10 @@ pub(crate) async fn standard_job_delete_many( /// Creates the client's queue record or refreshes its `updated_at`. pub(crate) async fn touch_queue(inner: &ClientInner, name: &str) -> Result { - let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session.storage(inner).queue_touch(name).await + let mut session = Session::begin(&inner.database, Access::Transaction).await?; + let queue = session.storage(inner).queue_touch(name).await?; + session.commit().await?; + Ok(queue) } /// Returns the IDs among `ids` of running jobs with a cancellation request. @@ -645,17 +682,21 @@ pub(crate) async fn job_cancel_requested( ids: &[i64], ) -> Result, Error> { let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session + let requested = session .storage(inner) .backend .job_cancel_requested(ids) - .await + .await?; + session.commit().await?; + Ok(requested) } /// Loads a queue record, if one exists. pub(crate) async fn load_queue(inner: &ClientInner, name: &str) -> Result, Error> { let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session.storage(inner).queue_get(name).await + let queue = session.storage(inner).queue_get(name).await?; + session.commit().await?; + Ok(queue) } /// Transactional job operations for a [`WorkContext`](crate::WorkContext), diff --git a/rust/riverqueue/src/storage/mysql.rs b/rust/riverqueue/src/storage/mysql.rs new file mode 100644 index 000000000..fab1e2415 --- /dev/null +++ b/rust/riverqueue/src/storage/mysql.rs @@ -0,0 +1,277 @@ +//! MySQL implementation of River's storage operations. + +use chrono::Utc; +use serde_json::{Map, Value}; +use sqlx::MySqlConnection; + +use super::Backend; +use crate::__private::DatabaseConnection; +use crate::database::mysql; +use crate::{Error, JobListParams, JobRow, JobState, Queue}; + +/// MySQL storage bound to one connection. +pub(super) struct MySqlBackend<'c> { + pub(super) connection: &'c mut MySqlConnection, +} + +impl MySqlBackend<'_> { + /// Lists jobs, optionally excluding running jobs before the limit + /// applies as bulk deletion does. + async fn list( + &mut self, + params: &JobListParams, + exclude_running: bool, + ) -> Result, Error> { + let kinds = params.kinds.iter().map(String::as_str).collect::>(); + let queues = params.queues.iter().map(String::as_str).collect::>(); + let tags_all = params + .tags_all + .iter() + .map(String::as_str) + .collect::>(); + let tags_any = params + .tags_any + .iter() + .map(String::as_str) + .collect::>(); + mysql::list( + self.connection, + &mysql::ListJobs { + exclude_running, + ids: ¶ms.ids, + keyset: params.keyset(), + kinds: &kinds, + limit: i32::try_from(params.limit).unwrap_or(i32::MAX), + metadata: params.metadata.as_ref(), + priorities: ¶ms.priorities, + queues: &queues, + states: ¶ms.states, + tags_all: &tags_all, + tags_any: &tags_any, + }, + ) + .await + .map_err(database_error) + } +} + +impl Backend for MySqlBackend<'_> { + fn connection(&mut self) -> DatabaseConnection<'_> { + DatabaseConnection::MySql(self.connection) + } + + async fn job_cancel(&mut self, id: i64) -> Result, Error> { + let updated = mysql::cancel(self.connection, id, Utc::now()) + .await + .map_err(database_error)?; + let Some(row) = updated else { + // Finalized jobs are returned unchanged, without a notification. + return self.job_get(id).await; + }; + let payload = serde_json::json!({ + "action": "cancel", + "job_id": id, + "queue": row.queue, + }) + .to_string(); + self.notify(crate::NOTIFICATION_TOPIC_CONTROL, &payload) + .await?; + Ok(Some(row)) + } + + // Like River Go, metadata that isn't valid JSON doesn't fail the lookup + // for the other jobs. + async fn job_cancel_requested(&mut self, ids: &[i64]) -> Result, Error> { + let ids = format!( + "[{}]", + ids.iter().map(i64::to_string).collect::>().join(",") + ); + Ok(sqlx::query_scalar( + "SELECT id FROM river_job \ + WHERE id IN (SELECT id FROM JSON_TABLE(?, '$[*]' COLUMNS(id BIGINT PATH '$')) AS wanted) \ + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') \ + AND state = 'running' \ + ORDER BY id", + ) + .bind(ids) + .fetch_all(&mut *self.connection) + .await?) + } + + async fn job_complete( + &mut self, + id: i64, + metadata_updates: &Map, + ) -> Result { + let Some(job) = mysql::get_locked(self.connection, id) + .await + .map_err(database_error)? + else { + return Err(Error::NotFound(crate::Record::Job(id))); + }; + if job.state != JobState::Running { + return Err(super::job_not_running(job.state)); + } + let now = Utc::now(); + mysql::complete( + self.connection, + &mysql::CompleteJob { + attempt: None, + error: None, + finalized_at: Some(now), + id, + metadata_updates: Some(metadata_updates), + now, + scheduled_at: None, + state: JobState::Completed, + }, + ) + .await + .map_err(database_error)? + .ok_or(Error::NotFound(crate::Record::Job(id))) + } + + async fn job_delete(&mut self, id: i64) -> Result { + if let Some(row) = mysql::delete(self.connection, id) + .await + .map_err(database_error)? + { + return Ok(row); + } + match mysql::get(self.connection, id) + .await + .map_err(database_error)? + { + Some(job) if job.state == JobState::Running => Err(Error::JobRunning), + None | Some(_) => Err(Error::NotFound(crate::Record::Job(id))), + } + } + + async fn job_delete_many(&mut self, filter: &JobListParams) -> Result, Error> { + // Running jobs are excluded before the limit applies, like Go. + // Each delete locks and rechecks the candidate's state. + let jobs = self.list(filter, true).await?; + let mut deleted = Vec::with_capacity(jobs.len()); + for job in jobs { + if let Some(row) = mysql::delete(self.connection, job.id) + .await + .map_err(database_error)? + { + deleted.push(row); + } + } + Ok(deleted) + } + + async fn job_claim( + &mut self, + id: i64, + client_id: &str, + max_attempted_by: i32, + ) -> Result, Error> { + let claimed = mysql::claim_selected( + self.connection, + &mysql::ClaimJobs { + client_id, + kinds: None, + limit: 1, + max_attempted_by, + now: Utc::now(), + queue: "", + }, + &[id], + ) + .await + .map_err(database_error)?; + Ok(claimed + .into_iter() + .next() + .and_then(crate::client::tolerant_row)) + } + + async fn job_get(&mut self, id: i64) -> Result, Error> { + mysql::get(self.connection, id) + .await + .map_err(database_error) + } + + async fn job_list(&mut self, params: &JobListParams) -> Result, Error> { + self.list(params, false).await + } + + async fn job_retry(&mut self, id: i64) -> Result, Error> { + let updated = mysql::retry(self.connection, id, Utc::now()) + .await + .map_err(database_error)?; + let Some(row) = updated else { + // Running and already-available jobs are returned unchanged. + return self.job_get(id).await; + }; + // Like Go's `JobRetry`, a retry sends no insert notification; + // producers find the job on their next poll. + Ok(Some(row)) + } + + async fn job_update( + &mut self, + id: i64, + metadata: &Map, + ) -> Result, Error> { + mysql::update(self.connection, id, metadata) + .await + .map_err(database_error) + } + + async fn notify(&mut self, topic: &str, payload: &str) -> Result<(), Error> { + mysql::notification_insert( + self.connection, + &[mysql::NotificationInput { payload, topic }], + ) + .await + .map_err(database_error)?; + Ok(()) + } + + async fn queue_get(&mut self, name: &str) -> Result, Error> { + mysql::queue_get(self.connection, name) + .await + .map_err(database_error) + } + + async fn queue_list(&mut self, limit: u32) -> Result, Error> { + mysql::queue_list(self.connection, i32::try_from(limit).unwrap_or(i32::MAX)) + .await + .map_err(database_error) + } + + async fn queue_set_paused(&mut self, name: &str, paused: bool) -> Result { + let now = Utc::now(); + let updated = if paused { + mysql::queue_pause(self.connection, name, now).await + } else { + mysql::queue_resume(self.connection, name, now).await + } + .map_err(database_error)?; + Ok(u64::try_from(updated.len()).unwrap_or(u64::MAX)) + } + + async fn queue_touch(&mut self, name: &str) -> Result { + mysql::queue_upsert(self.connection, name, &Map::new(), None, Utc::now()) + .await + .map_err(database_error) + } + + async fn queue_update( + &mut self, + name: &str, + metadata: Option<&Map>, + ) -> Result, Error> { + mysql::queue_update(self.connection, name, metadata, Utc::now()) + .await + .map_err(database_error) + } +} + +fn database_error(error: mysql::BackendError) -> Error { + Error::Database(error.into()) +} diff --git a/rust/riverqueue/tests/database_contract.rs b/rust/riverqueue/tests/database_contract.rs index b5ba2d2e1..dd2c264e5 100644 --- a/rust/riverqueue/tests/database_contract.rs +++ b/rust/riverqueue/tests/database_contract.rs @@ -28,6 +28,29 @@ fn sqlite_pool() -> SqlitePool { SqlitePoolOptions::new().connect_lazy_with(SqliteConnectOptions::new()) } +#[cfg(feature = "mysql")] +fn mysql_pool() -> sqlx::MySqlPool { + sqlx::mysql::MySqlPoolOptions::new().connect_lazy_with(sqlx::mysql::MySqlConnectOptions::new()) +} + +#[tokio::test] +#[cfg(feature = "mysql")] +async fn mysql_sources_and_transactions_implement_the_database_contract() { + use riverqueue::database::MySqlDatabase; + fn mysql_transaction(database: &Database, transaction: &mut Transaction<'_, sqlx::MySql>) { + assert_eq!(require_executor(database, transaction), DatabaseKind::MySql); + } + let _: for<'executor, 'transaction> fn( + &Database, + &'executor mut Transaction<'transaction, sqlx::MySql>, + ) = mysql_transaction; + let pool = mysql_pool(); + let database = require_source(MySqlDatabase::new(pool.clone())); + assert_eq!(database.kind(), DatabaseKind::MySql); + assert_eq!(database.postgres_schema(), None); + assert_eq!(require_source(pool).kind(), DatabaseKind::MySql); +} + fn require_executor<'executor, E: DatabaseTransactionExecutor<'executor>>( database: &Database, executor: E,