diff --git a/libsql-server/proto/namespace_fence.proto b/libsql-server/proto/namespace_fence.proto new file mode 100644 index 0000000000..efdee1ab81 --- /dev/null +++ b/libsql-server/proto/namespace_fence.proto @@ -0,0 +1,205 @@ +// Durable encoding of namespace fence records, command receipts and markers. +// +// See docs/NAMESPACE_FENCE.md. Every message here is stored, so fields are only ever added. +// Readers reject unknown enum values and missing required fields instead of guessing. +syntax = "proto3"; + +package namespace_fence; + +enum FenceRole { + FENCE_ROLE_UNSPECIFIED = 0; + FENCE_ROLE_SOURCE = 1; + FENCE_ROLE_TARGET = 2; +} + +// `UNFENCED` and `ABSENT` are never stored in a record; they appear as the `expected_state` of a +// request against a namespace that has no record. `UNKNOWN_UNAVAILABLE` is derived and never +// stored either. +enum FenceState { + FENCE_STATE_UNSPECIFIED = 0; + FENCE_STATE_UNFENCED = 1; + FENCE_STATE_ABSENT = 2; + FENCE_STATE_SOURCE_DRAINING = 3; + FENCE_STATE_SOURCE_WRITE_FENCED = 4; + FENCE_STATE_SOURCE_READ_DRAINING = 5; + FENCE_STATE_SOURCE_READ_FENCED = 6; + FENCE_STATE_RELEASED = 7; + FENCE_STATE_TARGET_QUARANTINED = 8; + FENCE_STATE_TARGET_IMPORT_DRAINING = 9; + FENCE_STATE_TARGET_VALIDATING = 10; + FENCE_STATE_TARGET_WRITE_FENCED = 11; + FENCE_STATE_TARGET_WRITABLE = 12; + FENCE_STATE_TARGET_ABORTED = 13; + FENCE_STATE_UNKNOWN_UNAVAILABLE = 14; +} + +enum CommandKind { + COMMAND_KIND_UNSPECIFIED = 0; + COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE = 1; + COMMAND_KIND_SET_SOURCE_READ_FENCE = 2; + COMMAND_KIND_CLEAR_SOURCE_READ_FENCE = 3; + COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE = 4; + COMMAND_KIND_CREATE_TARGET_QUARANTINED = 5; + COMMAND_KIND_SEAL_TARGET_IMPORT = 6; + COMMAND_KIND_RECORD_TARGET_VALIDATION = 7; + COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED = 8; + COMMAND_KIND_ENABLE_TARGET_WRITES = 9; + COMMAND_KIND_ABORT_QUARANTINED_TARGET = 10; + COMMAND_KIND_ADOPT_FENCE = 11; +} + +// Only the outcomes a receipt can record. Errors are never stored. +enum ReceiptOutcome { + RECEIPT_OUTCOME_UNSPECIFIED = 0; + RECEIPT_OUTCOME_APPLIED = 1; + RECEIPT_OUTCOME_ALREADY_APPLIED = 2; + RECEIPT_OUTCOME_DRAINING = 3; +} + +enum OnDeadline { + ON_DEADLINE_UNSPECIFIED = 0; + ON_DEADLINE_FAIL = 1; + ON_DEADLINE_FORCE_ROLLBACK = 2; +} + +enum ValidationResult { + VALIDATION_RESULT_UNSPECIFIED = 0; + VALIDATION_RESULT_OK = 1; + VALIDATION_RESULT_FAILED = 2; +} + +message DrainPolicy { + uint64 deadline_ms = 1; + OnDeadline on_deadline = 2; +} + +message FrozenBoundary { + string log_id = 1; + uint64 frame_no = 2; +} + +message LegacyBlocks { + bool block_reads = 1; + bool block_writes = 2; + optional string block_reason = 3; +} + +message ServerIdentity { + string build = 1; + string instance_id = 2; +} + +message TargetConfig { + optional uint64 max_db_size = 1; + optional string jwt_key = 2; + optional uint64 txn_timeout_s = 3; + bool allow_attach = 4; + optional string durability_mode = 5; + optional string bottomless_db_id = 6; +} + +message ValidationSnapshot { + string log_id = 1; + uint64 frame_no = 2; + uint64 page_count = 3; +} + +message ValidationRecord { + string operation_id = 1; + string command_id = 2; + ValidationResult result = 3; + string summary = 4; + optional ValidationSnapshot snapshot = 5; + int64 recorded_at_ms = 6; +} + +message Adoption { + string previous_operation_id = 1; + string new_operation_id = 2; + string command_id = 3; + repeated string approvers = 4; + string incident_ref = 5; + string reason = 6; + int64 at_ms = 7; + uint64 revision = 8; +} + +message FenceRecord { + string namespace = 1; + FenceRole role = 2; + FenceState state = 3; + uint64 revision = 4; + string operation_id = 5; + optional string log_id = 6; + optional string target_incarnation_id = 7; + optional DrainPolicy drain_policy = 8; + optional int64 drain_started_at_ms = 9; + optional FrozenBoundary frozen_boundary = 10; + optional ValidationRecord validation = 11; + LegacyBlocks legacy_blocks = 12; + int64 created_at_ms = 13; + int64 last_transition_at_ms = 14; + string last_command_id = 15; + ServerIdentity written_by = 16; + repeated Adoption adoptions = 17; +} + +message CommandReceipt { + string namespace = 1; + string operation_id = 2; + string command_id = 3; + CommandKind command = 4; + bytes fingerprint = 5; + ReceiptOutcome outcome = 6; + uint64 revision_before = 7; + uint64 revision_after = 8; + FenceState state_after = 9; + int64 applied_at_ms = 10; + string instance_id = 11; + optional Adoption adoption = 12; +} + +// Contents of `dbs//.fence`: a copy of the last committed record (or, for +// `CreateTargetQuarantined`, of the record about to be committed). +message FenceMarker { + uint32 format_version = 1; + FenceRecord record = 2; +} + +// Canonical input of a command fingerprint: everything in the request except `command_id`. +message FingerprintInput { + string namespace = 1; + string operation_id = 2; + CommandKind kind = 3; + FenceState expected_state = 4; + uint64 expected_revision = 5; + oneof args { + AcquireSourceWriteFenceArgs acquire_source_write_fence = 10; + DrainArgs set_source_read_fence = 11; + DrainArgs seal_target_import = 12; + TargetConfig create_target_quarantined = 13; + RecordTargetValidationArgs record_target_validation = 14; + AdoptFenceArgs adopt_fence = 15; + } +} + +message AcquireSourceWriteFenceArgs { + string expected_log_id = 1; + optional DrainPolicy drain_policy = 2; +} + +message DrainArgs { + optional DrainPolicy drain_policy = 1; +} + +message RecordTargetValidationArgs { + ValidationResult result = 1; + string summary = 2; +} + +message AdoptFenceArgs { + string current_operation_id = 1; + repeated string approvers = 2; + string incident_ref = 3; + string reason = 4; +} diff --git a/libsql-server/src/config.rs b/libsql-server/src/config.rs index 2c3c302a6d..9ac7add98b 100644 --- a/libsql-server/src/config.rs +++ b/libsql-server/src/config.rs @@ -187,6 +187,12 @@ pub struct MetaStoreConfig { pub allow_recover_from_fs: bool, /// Destroy the metastore if there is a restore error pub destroy_on_error: bool, + /// Allow namespace fences to be used: creates the fence tables. Fences that already exist + /// are loaded and enforced whether or not this is set. + pub namespace_fence: bool, + /// How long receipts of finished fence operations are kept. `None` is the default of + /// 30 days. + pub namespace_fence_receipt_retention: Option, } #[derive(Debug, Clone)] diff --git a/libsql-server/src/connection/program.rs b/libsql-server/src/connection/program.rs index 08dd9526f3..4d5ada51ff 100644 --- a/libsql-server/src/connection/program.rs +++ b/libsql-server/src/connection/program.rs @@ -370,7 +370,13 @@ pub async fn check_program_auth( } StmtKind::Attach(ref ns) => { ctx.auth.has_right(ns, Permission::AttachRead)?; - if !ctx.meta_store.handle(ns.clone()).await.get().allow_attach { + // A non-creating lookup: a missing namespace does not allow attach, and one + // whose fence state is not established is refused with its fence error. + let allow_attach = match ctx.meta_store.lookup(ns).await? { + Some(handle) => handle.get().allow_attach, + None => false, + }; + if !allow_attach { return Err(Error::Forbidden(format!( "Namespace `{ns}` doesn't allow attach" ))); diff --git a/libsql-server/src/error.rs b/libsql-server/src/error.rs index bfe67f47c7..f0cb631769 100644 --- a/libsql-server/src/error.rs +++ b/libsql-server/src/error.rs @@ -128,6 +128,8 @@ pub enum Error { RuntimeTaskJoinError(#[from] tokio::task::JoinError), #[error("database is not a primary")] NotAPrimary, + #[error(transparent)] + NamespaceFence(#[from] crate::namespace::fence::outcome::FenceError), } impl AsRef for Error { @@ -224,6 +226,7 @@ impl IntoResponse for &Error { AttachInMigration => self.format_err(StatusCode::BAD_REQUEST), RuntimeTaskJoinError(_) => self.format_err(StatusCode::INTERNAL_SERVER_ERROR), NotAPrimary => self.format_err(StatusCode::BAD_REQUEST), + NamespaceFence(e) => self.format_err(e.outcome().admin_http_status()), } } } diff --git a/libsql-server/src/generated/namespace_fence.rs b/libsql-server/src/generated/namespace_fence.rs new file mode 100644 index 0000000000..dcbbcafe96 --- /dev/null +++ b/libsql-server/src/generated/namespace_fence.rs @@ -0,0 +1,507 @@ +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DrainPolicy { + #[prost(uint64, tag = "1")] + pub deadline_ms: u64, + #[prost(enumeration = "OnDeadline", tag = "2")] + pub on_deadline: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FrozenBoundary { + #[prost(string, tag = "1")] + pub log_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LegacyBlocks { + #[prost(bool, tag = "1")] + pub block_reads: bool, + #[prost(bool, tag = "2")] + pub block_writes: bool, + #[prost(string, optional, tag = "3")] + pub block_reason: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ServerIdentity { + #[prost(string, tag = "1")] + pub build: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub instance_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TargetConfig { + #[prost(uint64, optional, tag = "1")] + pub max_db_size: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub jwt_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint64, optional, tag = "3")] + pub txn_timeout_s: ::core::option::Option, + #[prost(bool, tag = "4")] + pub allow_attach: bool, + #[prost(string, optional, tag = "5")] + pub durability_mode: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub bottomless_db_id: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidationSnapshot { + #[prost(string, tag = "1")] + pub log_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, + #[prost(uint64, tag = "3")] + pub page_count: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidationRecord { + #[prost(string, tag = "1")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub command_id: ::prost::alloc::string::String, + #[prost(enumeration = "ValidationResult", tag = "3")] + pub result: i32, + #[prost(string, tag = "4")] + pub summary: ::prost::alloc::string::String, + #[prost(message, optional, tag = "5")] + pub snapshot: ::core::option::Option, + #[prost(int64, tag = "6")] + pub recorded_at_ms: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Adoption { + #[prost(string, tag = "1")] + pub previous_operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub new_operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub command_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "4")] + pub approvers: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "5")] + pub incident_ref: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub reason: ::prost::alloc::string::String, + #[prost(int64, tag = "7")] + pub at_ms: i64, + #[prost(uint64, tag = "8")] + pub revision: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FenceRecord { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(enumeration = "FenceRole", tag = "2")] + pub role: i32, + #[prost(enumeration = "FenceState", tag = "3")] + pub state: i32, + #[prost(uint64, tag = "4")] + pub revision: u64, + #[prost(string, tag = "5")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, optional, tag = "6")] + pub log_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub target_incarnation_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag = "8")] + pub drain_policy: ::core::option::Option, + #[prost(int64, optional, tag = "9")] + pub drain_started_at_ms: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub frozen_boundary: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub validation: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub legacy_blocks: ::core::option::Option, + #[prost(int64, tag = "13")] + pub created_at_ms: i64, + #[prost(int64, tag = "14")] + pub last_transition_at_ms: i64, + #[prost(string, tag = "15")] + pub last_command_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "16")] + pub written_by: ::core::option::Option, + #[prost(message, repeated, tag = "17")] + pub adoptions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommandReceipt { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub command_id: ::prost::alloc::string::String, + #[prost(enumeration = "CommandKind", tag = "4")] + pub command: i32, + #[prost(bytes = "vec", tag = "5")] + pub fingerprint: ::prost::alloc::vec::Vec, + #[prost(enumeration = "ReceiptOutcome", tag = "6")] + pub outcome: i32, + #[prost(uint64, tag = "7")] + pub revision_before: u64, + #[prost(uint64, tag = "8")] + pub revision_after: u64, + #[prost(enumeration = "FenceState", tag = "9")] + pub state_after: i32, + #[prost(int64, tag = "10")] + pub applied_at_ms: i64, + #[prost(string, tag = "11")] + pub instance_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "12")] + pub adoption: ::core::option::Option, +} +/// Contents of `dbs//.fence`: a copy of the last committed record (or, for +/// `CreateTargetQuarantined`, of the record about to be committed). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FenceMarker { + #[prost(uint32, tag = "1")] + pub format_version: u32, + #[prost(message, optional, tag = "2")] + pub record: ::core::option::Option, +} +/// Canonical input of a command fingerprint: everything in the request except `command_id`. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FingerprintInput { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub operation_id: ::prost::alloc::string::String, + #[prost(enumeration = "CommandKind", tag = "3")] + pub kind: i32, + #[prost(enumeration = "FenceState", tag = "4")] + pub expected_state: i32, + #[prost(uint64, tag = "5")] + pub expected_revision: u64, + #[prost(oneof = "fingerprint_input::Args", tags = "10, 11, 12, 13, 14, 15")] + pub args: ::core::option::Option, +} +/// Nested message and enum types in `FingerprintInput`. +pub mod fingerprint_input { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Args { + #[prost(message, tag = "10")] + AcquireSourceWriteFence(super::AcquireSourceWriteFenceArgs), + #[prost(message, tag = "11")] + SetSourceReadFence(super::DrainArgs), + #[prost(message, tag = "12")] + SealTargetImport(super::DrainArgs), + #[prost(message, tag = "13")] + CreateTargetQuarantined(super::TargetConfig), + #[prost(message, tag = "14")] + RecordTargetValidation(super::RecordTargetValidationArgs), + #[prost(message, tag = "15")] + AdoptFence(super::AdoptFenceArgs), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AcquireSourceWriteFenceArgs { + #[prost(string, tag = "1")] + pub expected_log_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub drain_policy: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DrainArgs { + #[prost(message, optional, tag = "1")] + pub drain_policy: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RecordTargetValidationArgs { + #[prost(enumeration = "ValidationResult", tag = "1")] + pub result: i32, + #[prost(string, tag = "2")] + pub summary: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdoptFenceArgs { + #[prost(string, tag = "1")] + pub current_operation_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "2")] + pub approvers: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub incident_ref: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FenceRole { + Unspecified = 0, + Source = 1, + Target = 2, +} +impl FenceRole { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FenceRole::Unspecified => "FENCE_ROLE_UNSPECIFIED", + FenceRole::Source => "FENCE_ROLE_SOURCE", + FenceRole::Target => "FENCE_ROLE_TARGET", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FENCE_ROLE_UNSPECIFIED" => Some(Self::Unspecified), + "FENCE_ROLE_SOURCE" => Some(Self::Source), + "FENCE_ROLE_TARGET" => Some(Self::Target), + _ => None, + } + } +} +/// `UNFENCED` and `ABSENT` are never stored in a record; they appear as the `expected_state` of a +/// request against a namespace that has no record. `UNKNOWN_UNAVAILABLE` is derived and never +/// stored either. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FenceState { + Unspecified = 0, + Unfenced = 1, + Absent = 2, + SourceDraining = 3, + SourceWriteFenced = 4, + SourceReadDraining = 5, + SourceReadFenced = 6, + Released = 7, + TargetQuarantined = 8, + TargetImportDraining = 9, + TargetValidating = 10, + TargetWriteFenced = 11, + TargetWritable = 12, + TargetAborted = 13, + UnknownUnavailable = 14, +} +impl FenceState { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FenceState::Unspecified => "FENCE_STATE_UNSPECIFIED", + FenceState::Unfenced => "FENCE_STATE_UNFENCED", + FenceState::Absent => "FENCE_STATE_ABSENT", + FenceState::SourceDraining => "FENCE_STATE_SOURCE_DRAINING", + FenceState::SourceWriteFenced => "FENCE_STATE_SOURCE_WRITE_FENCED", + FenceState::SourceReadDraining => "FENCE_STATE_SOURCE_READ_DRAINING", + FenceState::SourceReadFenced => "FENCE_STATE_SOURCE_READ_FENCED", + FenceState::Released => "FENCE_STATE_RELEASED", + FenceState::TargetQuarantined => "FENCE_STATE_TARGET_QUARANTINED", + FenceState::TargetImportDraining => "FENCE_STATE_TARGET_IMPORT_DRAINING", + FenceState::TargetValidating => "FENCE_STATE_TARGET_VALIDATING", + FenceState::TargetWriteFenced => "FENCE_STATE_TARGET_WRITE_FENCED", + FenceState::TargetWritable => "FENCE_STATE_TARGET_WRITABLE", + FenceState::TargetAborted => "FENCE_STATE_TARGET_ABORTED", + FenceState::UnknownUnavailable => "FENCE_STATE_UNKNOWN_UNAVAILABLE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FENCE_STATE_UNSPECIFIED" => Some(Self::Unspecified), + "FENCE_STATE_UNFENCED" => Some(Self::Unfenced), + "FENCE_STATE_ABSENT" => Some(Self::Absent), + "FENCE_STATE_SOURCE_DRAINING" => Some(Self::SourceDraining), + "FENCE_STATE_SOURCE_WRITE_FENCED" => Some(Self::SourceWriteFenced), + "FENCE_STATE_SOURCE_READ_DRAINING" => Some(Self::SourceReadDraining), + "FENCE_STATE_SOURCE_READ_FENCED" => Some(Self::SourceReadFenced), + "FENCE_STATE_RELEASED" => Some(Self::Released), + "FENCE_STATE_TARGET_QUARANTINED" => Some(Self::TargetQuarantined), + "FENCE_STATE_TARGET_IMPORT_DRAINING" => Some(Self::TargetImportDraining), + "FENCE_STATE_TARGET_VALIDATING" => Some(Self::TargetValidating), + "FENCE_STATE_TARGET_WRITE_FENCED" => Some(Self::TargetWriteFenced), + "FENCE_STATE_TARGET_WRITABLE" => Some(Self::TargetWritable), + "FENCE_STATE_TARGET_ABORTED" => Some(Self::TargetAborted), + "FENCE_STATE_UNKNOWN_UNAVAILABLE" => Some(Self::UnknownUnavailable), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CommandKind { + Unspecified = 0, + AcquireSourceWriteFence = 1, + SetSourceReadFence = 2, + ClearSourceReadFence = 3, + ReleaseSourceWriteFence = 4, + CreateTargetQuarantined = 5, + SealTargetImport = 6, + RecordTargetValidation = 7, + PublishTargetReadableWriteFenced = 8, + EnableTargetWrites = 9, + AbortQuarantinedTarget = 10, + AdoptFence = 11, +} +impl CommandKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + CommandKind::Unspecified => "COMMAND_KIND_UNSPECIFIED", + CommandKind::AcquireSourceWriteFence => { + "COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE" + } + CommandKind::SetSourceReadFence => "COMMAND_KIND_SET_SOURCE_READ_FENCE", + CommandKind::ClearSourceReadFence => "COMMAND_KIND_CLEAR_SOURCE_READ_FENCE", + CommandKind::ReleaseSourceWriteFence => { + "COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE" + } + CommandKind::CreateTargetQuarantined => { + "COMMAND_KIND_CREATE_TARGET_QUARANTINED" + } + CommandKind::SealTargetImport => "COMMAND_KIND_SEAL_TARGET_IMPORT", + CommandKind::RecordTargetValidation => { + "COMMAND_KIND_RECORD_TARGET_VALIDATION" + } + CommandKind::PublishTargetReadableWriteFenced => { + "COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED" + } + CommandKind::EnableTargetWrites => "COMMAND_KIND_ENABLE_TARGET_WRITES", + CommandKind::AbortQuarantinedTarget => { + "COMMAND_KIND_ABORT_QUARANTINED_TARGET" + } + CommandKind::AdoptFence => "COMMAND_KIND_ADOPT_FENCE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "COMMAND_KIND_UNSPECIFIED" => Some(Self::Unspecified), + "COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE" => { + Some(Self::AcquireSourceWriteFence) + } + "COMMAND_KIND_SET_SOURCE_READ_FENCE" => Some(Self::SetSourceReadFence), + "COMMAND_KIND_CLEAR_SOURCE_READ_FENCE" => Some(Self::ClearSourceReadFence), + "COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE" => { + Some(Self::ReleaseSourceWriteFence) + } + "COMMAND_KIND_CREATE_TARGET_QUARANTINED" => { + Some(Self::CreateTargetQuarantined) + } + "COMMAND_KIND_SEAL_TARGET_IMPORT" => Some(Self::SealTargetImport), + "COMMAND_KIND_RECORD_TARGET_VALIDATION" => Some(Self::RecordTargetValidation), + "COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED" => { + Some(Self::PublishTargetReadableWriteFenced) + } + "COMMAND_KIND_ENABLE_TARGET_WRITES" => Some(Self::EnableTargetWrites), + "COMMAND_KIND_ABORT_QUARANTINED_TARGET" => Some(Self::AbortQuarantinedTarget), + "COMMAND_KIND_ADOPT_FENCE" => Some(Self::AdoptFence), + _ => None, + } + } +} +/// Only the outcomes a receipt can record. Errors are never stored. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ReceiptOutcome { + Unspecified = 0, + Applied = 1, + AlreadyApplied = 2, + Draining = 3, +} +impl ReceiptOutcome { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ReceiptOutcome::Unspecified => "RECEIPT_OUTCOME_UNSPECIFIED", + ReceiptOutcome::Applied => "RECEIPT_OUTCOME_APPLIED", + ReceiptOutcome::AlreadyApplied => "RECEIPT_OUTCOME_ALREADY_APPLIED", + ReceiptOutcome::Draining => "RECEIPT_OUTCOME_DRAINING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "RECEIPT_OUTCOME_UNSPECIFIED" => Some(Self::Unspecified), + "RECEIPT_OUTCOME_APPLIED" => Some(Self::Applied), + "RECEIPT_OUTCOME_ALREADY_APPLIED" => Some(Self::AlreadyApplied), + "RECEIPT_OUTCOME_DRAINING" => Some(Self::Draining), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum OnDeadline { + Unspecified = 0, + Fail = 1, + ForceRollback = 2, +} +impl OnDeadline { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + OnDeadline::Unspecified => "ON_DEADLINE_UNSPECIFIED", + OnDeadline::Fail => "ON_DEADLINE_FAIL", + OnDeadline::ForceRollback => "ON_DEADLINE_FORCE_ROLLBACK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ON_DEADLINE_UNSPECIFIED" => Some(Self::Unspecified), + "ON_DEADLINE_FAIL" => Some(Self::Fail), + "ON_DEADLINE_FORCE_ROLLBACK" => Some(Self::ForceRollback), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ValidationResult { + Unspecified = 0, + Ok = 1, + Failed = 2, +} +impl ValidationResult { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ValidationResult::Unspecified => "VALIDATION_RESULT_UNSPECIFIED", + ValidationResult::Ok => "VALIDATION_RESULT_OK", + ValidationResult::Failed => "VALIDATION_RESULT_FAILED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "VALIDATION_RESULT_UNSPECIFIED" => Some(Self::Unspecified), + "VALIDATION_RESULT_OK" => Some(Self::Ok), + "VALIDATION_RESULT_FAILED" => Some(Self::Failed), + _ => None, + } + } +} diff --git a/libsql-server/src/main.rs b/libsql-server/src/main.rs index 307d5482fe..5d738a1ed5 100644 --- a/libsql-server/src/main.rs +++ b/libsql-server/src/main.rs @@ -258,6 +258,16 @@ struct Cli { #[clap(long, env = "SQLD_ALLOW_METASTORE_RECOVERY")] allow_metastore_recovery: bool, + /// Allow namespace fences to be used (see `docs/NAMESPACE_FENCE.md`). Off by default. + /// Fences that already exist in the metastore are enforced either way. + #[clap(long, env = "SQLD_ENABLE_NAMESPACE_FENCE")] + enable_namespace_fence: bool, + + /// How long, in seconds, receipts of finished namespace-fence operations are kept. + /// Defaults to 30 days. + #[clap(long, env = "SQLD_NAMESPACE_FENCE_RECEIPT_RETENTION_S")] + namespace_fence_receipt_retention_s: Option, + /// Shutdown timeout duration in seconds, defaults to 30 seconds. #[clap(long, env = "SQLD_SHUTDOWN_TIMEOUT")] shutdown_timeout: Option, @@ -650,6 +660,10 @@ fn make_meta_store_config(config: &Cli) -> anyhow::Result { bottomless, allow_recover_from_fs: config.allow_metastore_recovery, destroy_on_error: config.meta_store_destroy_on_error, + namespace_fence: config.enable_namespace_fence, + namespace_fence_receipt_retention: config + .namespace_fence_receipt_retention_s + .map(Duration::from_secs), }) } diff --git a/libsql-server/src/namespace/fence/command.rs b/libsql-server/src/namespace/fence/command.rs new file mode 100644 index 0000000000..718fc78d83 --- /dev/null +++ b/libsql-server/src/namespace/fence/command.rs @@ -0,0 +1,420 @@ +//! Fence commands, requests and their canonical fingerprint (`docs/NAMESPACE_FENCE.md` +//! sections 4.2, 4.4 and 5.3). + +use std::fmt; + +use prost::Message as _; +use sha2::{Digest as _, Sha256}; +use uuid::Uuid; + +use crate::namespace::NamespaceName; + +use super::proto; +use super::record::codec; +use super::state::FenceState; + +/// Largest accepted `RecordTargetValidation` summary, in bytes. +pub const MAX_VALIDATION_SUMMARY_BYTES: usize = 4096; + +/// What happens when a drain deadline passes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OnDeadline { + /// Answer `DRAINING`; the durable state stays draining and admission stays closed. + Fail, + /// Roll back (or cancel, for reads) the work still holding the drain open, then keep + /// waiting for it to actually end. + ForceRollback, +} + +impl OnDeadline { + pub const fn as_str(self) -> &'static str { + match self { + OnDeadline::Fail => "fail", + OnDeadline::ForceRollback => "force_rollback", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DrainPolicy { + pub deadline_ms: u64, + pub on_deadline: OnDeadline, +} + +/// The subset of namespace configuration `CreateTargetQuarantined` accepts. Restore options +/// and dump URLs are not part of it: import goes through the migration capability. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct TargetConfig { + pub max_db_size: Option, + pub jwt_key: Option, + pub txn_timeout_s: Option, + pub allow_attach: bool, + pub durability_mode: Option, + pub bottomless_db_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ValidationResult { + Ok, + Failed, +} + +impl ValidationResult { + pub const fn as_str(self) -> &'static str { + match self { + ValidationResult::Ok => "ok", + ValidationResult::Failed => "failed", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AdoptArgs { + /// The operation that currently owns the record, as the adopter believes it. + pub current_operation_id: Uuid, + /// Two distinct, non-empty identities. Recorded, not verified (section 12). + pub approvers: Vec, + pub incident_ref: String, + pub reason: String, +} + +/// A mutating fence command and its command-specific arguments. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum FenceCommand { + AcquireSourceWriteFence { + /// The replication log id the caller observed on the source. + expected_log_id: Uuid, + drain_policy: Option, + }, + SetSourceReadFence { + drain_policy: Option, + }, + ClearSourceReadFence, + ReleaseSourceWriteFence, + CreateTargetQuarantined { + config: TargetConfig, + }, + SealTargetImport { + drain_policy: Option, + }, + RecordTargetValidation { + result: ValidationResult, + summary: String, + }, + PublishTargetReadableWriteFenced, + EnableTargetWrites, + AbortQuarantinedTarget, + AdoptFence(AdoptArgs), +} + +/// The kind of a command, without its arguments. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CommandKind { + AcquireSourceWriteFence, + SetSourceReadFence, + ClearSourceReadFence, + ReleaseSourceWriteFence, + CreateTargetQuarantined, + SealTargetImport, + RecordTargetValidation, + PublishTargetReadableWriteFenced, + EnableTargetWrites, + AbortQuarantinedTarget, + AdoptFence, +} + +impl CommandKind { + pub const ALL: [CommandKind; 11] = [ + CommandKind::AcquireSourceWriteFence, + CommandKind::SetSourceReadFence, + CommandKind::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined, + CommandKind::SealTargetImport, + CommandKind::RecordTargetValidation, + CommandKind::PublishTargetReadableWriteFenced, + CommandKind::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget, + CommandKind::AdoptFence, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + CommandKind::AcquireSourceWriteFence => "AcquireSourceWriteFence", + CommandKind::SetSourceReadFence => "SetSourceReadFence", + CommandKind::ClearSourceReadFence => "ClearSourceReadFence", + CommandKind::ReleaseSourceWriteFence => "ReleaseSourceWriteFence", + CommandKind::CreateTargetQuarantined => "CreateTargetQuarantined", + CommandKind::SealTargetImport => "SealTargetImport", + CommandKind::RecordTargetValidation => "RecordTargetValidation", + CommandKind::PublishTargetReadableWriteFenced => "PublishTargetReadableWriteFenced", + CommandKind::EnableTargetWrites => "EnableTargetWrites", + CommandKind::AbortQuarantinedTarget => "AbortQuarantinedTarget", + CommandKind::AdoptFence => "AdoptFence", + } + } + + /// The state a successful command finally leaves the record in, where that is a single + /// state. A command from the owner whose goal state the record is already in is + /// `ALREADY_APPLIED`. `RecordTargetValidation` and `AdoptFence` do not move the state and + /// have none. + pub const fn goal_state(self) -> Option { + match self { + CommandKind::AcquireSourceWriteFence => Some(FenceState::SourceWriteFenced), + CommandKind::SetSourceReadFence => Some(FenceState::SourceReadFenced), + CommandKind::ClearSourceReadFence => Some(FenceState::SourceWriteFenced), + CommandKind::ReleaseSourceWriteFence => Some(FenceState::Released), + CommandKind::CreateTargetQuarantined => Some(FenceState::TargetQuarantined), + CommandKind::SealTargetImport => Some(FenceState::TargetValidating), + CommandKind::PublishTargetReadableWriteFenced => Some(FenceState::TargetWriteFenced), + CommandKind::EnableTargetWrites => Some(FenceState::TargetWritable), + CommandKind::AbortQuarantinedTarget => Some(FenceState::TargetAborted), + CommandKind::RecordTargetValidation | CommandKind::AdoptFence => None, + } + } +} + +impl fmt::Display for CommandKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FenceCommand { + pub fn kind(&self) -> CommandKind { + match self { + FenceCommand::AcquireSourceWriteFence { .. } => CommandKind::AcquireSourceWriteFence, + FenceCommand::SetSourceReadFence { .. } => CommandKind::SetSourceReadFence, + FenceCommand::ClearSourceReadFence => CommandKind::ClearSourceReadFence, + FenceCommand::ReleaseSourceWriteFence => CommandKind::ReleaseSourceWriteFence, + FenceCommand::CreateTargetQuarantined { .. } => CommandKind::CreateTargetQuarantined, + FenceCommand::SealTargetImport { .. } => CommandKind::SealTargetImport, + FenceCommand::RecordTargetValidation { .. } => CommandKind::RecordTargetValidation, + FenceCommand::PublishTargetReadableWriteFenced => { + CommandKind::PublishTargetReadableWriteFenced + } + FenceCommand::EnableTargetWrites => CommandKind::EnableTargetWrites, + FenceCommand::AbortQuarantinedTarget => CommandKind::AbortQuarantinedTarget, + FenceCommand::AdoptFence(_) => CommandKind::AdoptFence, + } + } +} + +/// One mutating request: the common fields of section 4.2 and the command. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FenceRequest { + pub namespace: NamespaceName, + pub operation_id: Uuid, + pub command_id: Uuid, + /// `Unfenced` or `Absent` for a namespace with no record. + pub expected_state: FenceState, + /// `0` for a namespace with no record. + pub expected_revision: u64, + pub command: FenceCommand, +} + +impl FenceRequest { + /// SHA-256 of the deterministic protobuf encoding of everything in the request except + /// `command_id` (section 5.3). Two requests with the same `(operation_id, command_id)` and + /// different fingerprints are a `FENCE_COMMAND_CONFLICT`. + pub fn fingerprint(&self) -> Fingerprint { + let input = proto::FingerprintInput { + namespace: self.namespace.as_str().to_string(), + operation_id: self.operation_id.to_string(), + kind: codec::command_kind_to_proto(self.command.kind()) as i32, + expected_state: codec::state_to_proto(self.expected_state) as i32, + expected_revision: self.expected_revision, + args: fingerprint_args(&self.command), + }; + Fingerprint(Sha256::digest(input.encode_to_vec()).into()) + } +} + +fn fingerprint_args(command: &FenceCommand) -> Option { + use proto::fingerprint_input::Args; + + let drain = |p: &Option| proto::DrainArgs { + drain_policy: p.as_ref().map(codec::drain_policy_to_proto), + }; + + match command { + FenceCommand::AcquireSourceWriteFence { + expected_log_id, + drain_policy, + } => Some(Args::AcquireSourceWriteFence( + proto::AcquireSourceWriteFenceArgs { + expected_log_id: expected_log_id.to_string(), + drain_policy: drain_policy.as_ref().map(codec::drain_policy_to_proto), + }, + )), + FenceCommand::SetSourceReadFence { drain_policy } => { + Some(Args::SetSourceReadFence(drain(drain_policy))) + } + FenceCommand::SealTargetImport { drain_policy } => { + Some(Args::SealTargetImport(drain(drain_policy))) + } + FenceCommand::CreateTargetQuarantined { config } => Some(Args::CreateTargetQuarantined( + codec::target_config_to_proto(config), + )), + FenceCommand::RecordTargetValidation { result, summary } => Some( + Args::RecordTargetValidation(proto::RecordTargetValidationArgs { + result: codec::validation_result_to_proto(*result) as i32, + summary: summary.clone(), + }), + ), + FenceCommand::AdoptFence(args) => Some(Args::AdoptFence(proto::AdoptFenceArgs { + current_operation_id: args.current_operation_id.to_string(), + approvers: args.approvers.clone(), + incident_ref: args.incident_ref.clone(), + reason: args.reason.clone(), + })), + FenceCommand::ClearSourceReadFence + | FenceCommand::ReleaseSourceWriteFence + | FenceCommand::PublishTargetReadableWriteFenced + | FenceCommand::EnableTargetWrites + | FenceCommand::AbortQuarantinedTarget => None, + } +} + +/// A canonical request fingerprint. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Fingerprint(pub [u8; 32]); + +impl Fingerprint { + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl fmt::Display for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("sha256:")?; + for b in self.0 { + write!(f, "{b:02x}")?; + } + Ok(()) + } +} + +impl fmt::Debug for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(command: FenceCommand) -> FenceRequest { + FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(2), + expected_state: FenceState::Unfenced, + expected_revision: 0, + command, + } + } + + fn acquire() -> FenceCommand { + FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::Fail, + }), + } + } + + #[test] + fn fingerprint_ignores_command_id() { + let a = request(acquire()); + let mut b = a.clone(); + b.command_id = Uuid::from_u128(99); + assert_eq!(a.fingerprint(), b.fingerprint()); + } + + #[test] + fn fingerprint_covers_every_other_field() { + let base = request(acquire()); + let fp = base.fingerprint(); + + let mut changed = Vec::new(); + let mut r = base.clone(); + r.namespace = NamespaceName::from("db2"); + changed.push(r); + let mut r = base.clone(); + r.operation_id = Uuid::from_u128(7); + changed.push(r); + let mut r = base.clone(); + r.expected_state = FenceState::Released; + changed.push(r); + let mut r = base.clone(); + r.expected_revision = 1; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(4), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::Fail, + }), + }; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::ForceRollback, + }), + }; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: None, + }; + changed.push(r); + + for r in changed { + assert_ne!(r.fingerprint(), fp, "{r:?}"); + } + } + + #[test] + fn fingerprint_distinguishes_argumentless_commands() { + let kinds = [ + FenceCommand::ClearSourceReadFence, + FenceCommand::ReleaseSourceWriteFence, + FenceCommand::PublishTargetReadableWriteFenced, + FenceCommand::EnableTargetWrites, + FenceCommand::AbortQuarantinedTarget, + FenceCommand::SetSourceReadFence { drain_policy: None }, + FenceCommand::SealTargetImport { drain_policy: None }, + ]; + let fps: std::collections::HashSet<_> = kinds + .into_iter() + .map(|c| request(c).fingerprint()) + .collect(); + assert_eq!(fps.len(), 7); + } + + /// The fingerprint is stored in receipts and compared on every replay, so its encoding must + /// never change: a change would turn every stored receipt into a command conflict. + #[test] + fn fingerprint_is_stable() { + assert_eq!( + request(acquire()).fingerprint().to_string(), + "sha256:c585d3570b5eb5a3ef9b8efb89eca26e2336c7e19a561fa3db667c4cde905515" + ); + } + + #[test] + fn every_kind_has_a_name() { + let names: std::collections::HashSet<_> = + CommandKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(names.len(), CommandKind::ALL.len()); + } +} diff --git a/libsql-server/src/namespace/fence/mod.rs b/libsql-server/src/namespace/fence/mod.rs new file mode 100644 index 0000000000..672b0565e8 --- /dev/null +++ b/libsql-server/src/namespace/fence/mod.rs @@ -0,0 +1,30 @@ +//! Namespace fence: a durable, operation-owned control record that an external operation (for +//! example, moving a database between servers) uses as the data-plane authority boundary for +//! one namespace. +//! +//! `docs/NAMESPACE_FENCE.md` is the contract and the design. This module holds the parts with +//! no I/O: the states and permission matrix ([`state`]), the stable outcome codes and their +//! protocol mappings ([`outcome`]), commands and their canonical fingerprint ([`command`]), +//! records, receipts and markers with their strict durable encoding ([`record`]), the pure +//! transition function ([`transition`]), and the metastore tables, compare-and-swap and marker +//! file that persist them ([`store`], driven by `MetaStore::apply_fence_command`). + +// The persistence, controller and protocol layers that consume these types land in the +// following commits of this series; until then most of the module is unused by the rest of +// the crate. This attribute is removed once they are wired. +#![allow(dead_code)] + +pub mod command; +pub mod outcome; +pub mod record; +pub mod state; +pub mod store; +pub mod transition; + +#[allow(clippy::all)] +pub(crate) mod proto { + include!("../../generated/namespace_fence.rs"); +} + +/// Version of the fence admin protocol reported by capability discovery. +pub const FENCE_PROTOCOL_VERSION: u32 = 1; diff --git a/libsql-server/src/namespace/fence/outcome.rs b/libsql-server/src/namespace/fence/outcome.rs new file mode 100644 index 0000000000..6a9b6a8b57 --- /dev/null +++ b/libsql-server/src/namespace/fence/outcome.rs @@ -0,0 +1,409 @@ +//! Stable outcome codes and their protocol mappings (`docs/NAMESPACE_FENCE.md` section 6). + +use std::fmt; +use std::str::FromStr; + +use hyper::StatusCode; + +/// gRPC metadata key carrying the stable code of a fence denial. +pub const GRPC_FENCE_CODE_METADATA: &str = "x-libsql-fence-code"; + +/// Every machine-readable outcome a fence command or a fenced data-plane operation can report. +/// Clients match on [`FenceOutcome::as_str`], never on a message. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceOutcome { + Applied, + AlreadyApplied, + Draining, + MigrationWriteFenced, + MigrationReadFenced, + MigrationTargetQuarantined, + FenceStateUnavailable, + OperationCapabilityRequired, + FenceOwnedByAnotherOperation, + FenceRevisionMismatch, + InvalidFenceTransition, + FenceCommandConflict, + FenceCommitIndeterminate, + FencePreconditionFailed, +} + +/// What kind of answer an outcome is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutcomeKind { + Success, + InProgress, + /// A denial of ordinary data-plane or lifecycle work because of the fence. + DataPlane, + /// A refusal of a fence command (or of capability work). + Control, +} + +impl FenceOutcome { + pub const ALL: [FenceOutcome; 14] = [ + FenceOutcome::Applied, + FenceOutcome::AlreadyApplied, + FenceOutcome::Draining, + FenceOutcome::MigrationWriteFenced, + FenceOutcome::MigrationReadFenced, + FenceOutcome::MigrationTargetQuarantined, + FenceOutcome::FenceStateUnavailable, + FenceOutcome::OperationCapabilityRequired, + FenceOutcome::FenceOwnedByAnotherOperation, + FenceOutcome::FenceRevisionMismatch, + FenceOutcome::InvalidFenceTransition, + FenceOutcome::FenceCommandConflict, + FenceOutcome::FenceCommitIndeterminate, + FenceOutcome::FencePreconditionFailed, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + FenceOutcome::Applied => "APPLIED", + FenceOutcome::AlreadyApplied => "ALREADY_APPLIED", + FenceOutcome::Draining => "DRAINING", + FenceOutcome::MigrationWriteFenced => "MIGRATION_WRITE_FENCED", + FenceOutcome::MigrationReadFenced => "MIGRATION_READ_FENCED", + FenceOutcome::MigrationTargetQuarantined => "MIGRATION_TARGET_QUARANTINED", + FenceOutcome::FenceStateUnavailable => "FENCE_STATE_UNAVAILABLE", + FenceOutcome::OperationCapabilityRequired => "OPERATION_CAPABILITY_REQUIRED", + FenceOutcome::FenceOwnedByAnotherOperation => "FENCE_OWNED_BY_ANOTHER_OPERATION", + FenceOutcome::FenceRevisionMismatch => "FENCE_REVISION_MISMATCH", + FenceOutcome::InvalidFenceTransition => "INVALID_FENCE_TRANSITION", + FenceOutcome::FenceCommandConflict => "FENCE_COMMAND_CONFLICT", + FenceOutcome::FenceCommitIndeterminate => "FENCE_COMMIT_INDETERMINATE", + FenceOutcome::FencePreconditionFailed => "FENCE_PRECONDITION_FAILED", + } + } + + pub const fn kind(self) -> OutcomeKind { + match self { + FenceOutcome::Applied | FenceOutcome::AlreadyApplied => OutcomeKind::Success, + FenceOutcome::Draining => OutcomeKind::InProgress, + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable => OutcomeKind::DataPlane, + FenceOutcome::OperationCapabilityRequired + | FenceOutcome::FenceOwnedByAnotherOperation + | FenceOutcome::FenceRevisionMismatch + | FenceOutcome::InvalidFenceTransition + | FenceOutcome::FenceCommandConflict + | FenceOutcome::FenceCommitIndeterminate + | FenceOutcome::FencePreconditionFailed => OutcomeKind::Control, + } + } + + pub const fn is_error(self) -> bool { + matches!(self.kind(), OutcomeKind::DataPlane | OutcomeKind::Control) + } + + /// Status code on the admin API. + pub fn admin_http_status(self) -> StatusCode { + match self { + FenceOutcome::Applied | FenceOutcome::AlreadyApplied => StatusCode::OK, + FenceOutcome::Draining => StatusCode::ACCEPTED, + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable => StatusCode::LOCKED, + FenceOutcome::OperationCapabilityRequired => StatusCode::FORBIDDEN, + FenceOutcome::FenceOwnedByAnotherOperation + | FenceOutcome::FenceRevisionMismatch + | FenceOutcome::InvalidFenceTransition + | FenceOutcome::FenceCommandConflict + | FenceOutcome::FenceCommitIndeterminate => StatusCode::CONFLICT, + FenceOutcome::FencePreconditionFailed => StatusCode::PRECONDITION_FAILED, + } + } + + /// Status code on the user HTTP API (`/`, `/v1`, `/v2`, `/v3`, `/dump`). Only data-plane + /// denials reach it. + pub fn user_http_status(self) -> Option { + match self.kind() { + OutcomeKind::DataPlane => Some(StatusCode::LOCKED), + _ => None, + } + } + + /// The Hrana error `code`. Only data-plane denials reach Hrana. + pub fn hrana_code(self) -> Option<&'static str> { + match self.kind() { + OutcomeKind::DataPlane => Some(self.as_str()), + _ => None, + } + } + + /// The gRPC status code on RPC, proxy connection and replication services. Never + /// `UNAVAILABLE`, which the write proxy retries without bound. + pub fn grpc_code(self) -> Option { + match self { + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable + | FenceOutcome::OperationCapabilityRequired => Some(tonic::Code::FailedPrecondition), + _ => None, + } + } + + /// The value of the proxy protocol's `Error.stable_code` field. + pub fn proxy_stable_code(self) -> Option<&'static str> { + match self.kind() { + OutcomeKind::DataPlane => Some(self.as_str()), + _ => None, + } + } +} + +impl fmt::Display for FenceOutcome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown fence outcome `{0}`")] +pub struct UnknownFenceOutcome(pub String); + +impl FromStr for FenceOutcome { + type Err = UnknownFenceOutcome; + + fn from_str(s: &str) -> Result { + FenceOutcome::ALL + .iter() + .copied() + .find(|o| o.as_str() == s) + .ok_or_else(|| UnknownFenceOutcome(s.to_string())) + } +} + +/// The bounded `detail` reason of an error outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceDetail { + // FENCE_PRECONDITION_FAILED + AdminAuthRequired, + FenceDisabled, + NotPrimary, + SharedSchemaUnsupported, + NamespaceIdentityMismatch, + NamespaceExists, + ValidationReceiptRequired, + RestoreNotAllowed, + AdoptionNotAuthorised, + InvalidArgument, + // INVALID_FENCE_TRANSITION + RoleMismatch, + OperationFinished, + // FENCE_STATE_UNAVAILABLE + CorruptRecord, + UnsupportedFormatVersion, + IncompleteTargetCreation, + MetastoreBehindMarker, + IndeterminateCommit, +} + +impl FenceDetail { + pub const fn as_str(self) -> &'static str { + match self { + FenceDetail::AdminAuthRequired => "admin_auth_required", + FenceDetail::FenceDisabled => "fence_disabled", + FenceDetail::NotPrimary => "not_primary", + FenceDetail::SharedSchemaUnsupported => "shared_schema_unsupported", + FenceDetail::NamespaceIdentityMismatch => "namespace_identity_mismatch", + FenceDetail::NamespaceExists => "namespace_exists", + FenceDetail::ValidationReceiptRequired => "validation_receipt_required", + FenceDetail::RestoreNotAllowed => "restore_not_allowed", + FenceDetail::AdoptionNotAuthorised => "adoption_not_authorised", + FenceDetail::InvalidArgument => "invalid_argument", + FenceDetail::RoleMismatch => "role_mismatch", + FenceDetail::OperationFinished => "operation_finished", + FenceDetail::CorruptRecord => "corrupt_record", + FenceDetail::UnsupportedFormatVersion => "unsupported_format_version", + FenceDetail::IncompleteTargetCreation => "incomplete_target_creation", + FenceDetail::MetastoreBehindMarker => "metastore_behind_marker", + FenceDetail::IndeterminateCommit => "indeterminate_commit", + } + } +} + +impl fmt::Display for FenceDetail { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// An error outcome with its bounded detail and a human message. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{outcome}: {message}")] +pub struct FenceError { + outcome: FenceOutcome, + detail: Option, + message: String, +} + +impl FenceError { + /// # Panics + /// + /// If `outcome` is not an error outcome. That is a programming error, never input. + pub fn new(outcome: FenceOutcome, message: impl Into) -> Self { + assert!(outcome.is_error(), "{outcome} is not an error outcome"); + Self { + outcome, + detail: None, + message: message.into(), + } + } + + pub fn with_detail(mut self, detail: FenceDetail) -> Self { + self.detail = Some(detail); + self + } + + pub fn outcome(&self) -> FenceOutcome { + self.outcome + } + + pub fn detail(&self) -> Option { + self.detail + } + + pub fn message(&self) -> &str { + &self.message + } + + /// A gRPC status for this error, if the outcome has a gRPC mapping. The code is in the + /// [`GRPC_FENCE_CODE_METADATA`] entry and prefixes the message. + pub fn to_grpc_status(&self) -> Option { + let code = self.outcome.grpc_code()?; + let mut status = tonic::Status::new(code, format!("{}: {}", self.outcome, self.message)); + status.metadata_mut().insert( + GRPC_FENCE_CODE_METADATA, + tonic::metadata::MetadataValue::from_static(self.outcome.as_str()), + ); + Some(status) + } + + /// The stable code carried by a gRPC status produced by [`FenceError::to_grpc_status`]. + pub fn outcome_from_grpc_status(status: &tonic::Status) -> Option { + status + .metadata() + .get(GRPC_FENCE_CODE_METADATA)? + .to_str() + .ok()? + .parse() + .ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes_round_trip() { + for outcome in FenceOutcome::ALL { + assert_eq!(outcome.as_str().parse::().unwrap(), outcome); + } + assert!("applied".parse::().is_err()); + } + + /// Section 6: data-plane denials are never 500, 503, 429 or gRPC UNAVAILABLE. + #[test] + fn denials_are_never_retryable_statuses() { + let retryable = [ + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::SERVICE_UNAVAILABLE, + StatusCode::TOO_MANY_REQUESTS, + StatusCode::BAD_GATEWAY, + StatusCode::GATEWAY_TIMEOUT, + ]; + for outcome in FenceOutcome::ALL { + assert!( + !retryable.contains(&outcome.admin_http_status()), + "{outcome}" + ); + if let Some(status) = outcome.user_http_status() { + assert!(!retryable.contains(&status), "{outcome}"); + } + assert_ne!(outcome.grpc_code(), Some(tonic::Code::Unavailable)); + } + } + + #[test] + fn protocol_table() { + use FenceOutcome as O; + let rows = [ + (O::Applied, 200, None, None), + (O::AlreadyApplied, 200, None, None), + (O::Draining, 202, None, None), + ( + O::MigrationWriteFenced, + 423, + Some(423), + Some("MIGRATION_WRITE_FENCED"), + ), + ( + O::MigrationReadFenced, + 423, + Some(423), + Some("MIGRATION_READ_FENCED"), + ), + ( + O::MigrationTargetQuarantined, + 423, + Some(423), + Some("MIGRATION_TARGET_QUARANTINED"), + ), + ( + O::FenceStateUnavailable, + 423, + Some(423), + Some("FENCE_STATE_UNAVAILABLE"), + ), + (O::OperationCapabilityRequired, 403, None, None), + (O::FenceOwnedByAnotherOperation, 409, None, None), + (O::FenceRevisionMismatch, 409, None, None), + (O::InvalidFenceTransition, 409, None, None), + (O::FenceCommandConflict, 409, None, None), + (O::FenceCommitIndeterminate, 409, None, None), + (O::FencePreconditionFailed, 412, None, None), + ]; + assert_eq!(rows.len(), FenceOutcome::ALL.len()); + for (outcome, admin, user, hrana) in rows { + assert_eq!(outcome.admin_http_status().as_u16(), admin, "{outcome}"); + assert_eq!( + outcome.user_http_status().map(|s| s.as_u16()), + user, + "{outcome}" + ); + assert_eq!(outcome.hrana_code(), hrana, "{outcome}"); + assert_eq!(outcome.proxy_stable_code(), hrana, "{outcome}"); + } + } + + #[test] + fn grpc_status_carries_code() { + let err = FenceError::new(FenceOutcome::MigrationReadFenced, "reads are fenced"); + let status = err.to_grpc_status().unwrap(); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert!(status.message().starts_with("MIGRATION_READ_FENCED: ")); + assert_eq!( + FenceError::outcome_from_grpc_status(&status), + Some(FenceOutcome::MigrationReadFenced) + ); + assert_eq!( + FenceError::outcome_from_grpc_status(&tonic::Status::unavailable("x")), + None + ); + + let control = FenceError::new(FenceOutcome::FenceRevisionMismatch, "stale"); + assert!(control.to_grpc_status().is_none()); + } + + #[test] + #[should_panic] + fn success_is_not_an_error() { + FenceError::new(FenceOutcome::Applied, "nope"); + } +} diff --git a/libsql-server/src/namespace/fence/record.rs b/libsql-server/src/namespace/fence/record.rs new file mode 100644 index 0000000000..a2d9f7f5dd --- /dev/null +++ b/libsql-server/src/namespace/fence/record.rs @@ -0,0 +1,908 @@ +//! Fence records, command receipts, the on-disk marker, and their durable encoding +//! (`docs/NAMESPACE_FENCE.md` sections 5.1, 5.2 and 5.6). +//! +//! Decoding is strict: an unknown format version, an unknown enum value, a missing required +//! field, a malformed id or a record that contradicts itself is an error, which the store turns +//! into `UNKNOWN_UNAVAILABLE`. Nothing is defaulted. + +use prost::Message as _; +use uuid::Uuid; + +use crate::namespace::NamespaceName; + +use super::command::{CommandKind, DrainPolicy, Fingerprint, TargetConfig, ValidationResult}; +use super::outcome::FenceOutcome; +use super::proto; +use super::state::{Admission, FenceState, Role}; + +/// The only `format_version` this server writes and reads. +pub const FENCE_FORMAT_VERSION: u32 = 1; + +/// Identity of the namespace copy a record is about. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NamespaceIdentity { + /// Replication log id: for a source, captured at acquisition and checked against the + /// caller's expectation; for a target, known once the namespace exists. + pub log_id: Option, + /// Server-generated id of a target created by `CreateTargetQuarantined`. + pub target_incarnation_id: Option, +} + +/// The source's replication position once no writer can commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrozenBoundary { + pub log_id: Uuid, + pub frame_no: u64, +} + +/// The pre-fence values of the legacy `block_*` configuration fields, restored when the +/// operation finishes (section 13.2). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LegacyBlocks { + pub block_reads: bool, + pub block_writes: bool, + pub block_reason: Option, +} + +/// The server process that wrote something. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerIdentity { + pub build: String, + pub instance_id: Uuid, +} + +/// What the server observed of a target when a validation result was recorded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ValidationSnapshot { + pub log_id: Uuid, + pub frame_no: u64, + pub page_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ValidationRecord { + pub operation_id: Uuid, + pub command_id: Uuid, + pub result: ValidationResult, + pub summary: String, + pub snapshot: Option, + pub recorded_at_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Adoption { + pub previous_operation_id: Uuid, + pub new_operation_id: Uuid, + pub command_id: Uuid, + pub approvers: Vec, + pub incident_ref: String, + pub reason: String, + pub at_ms: i64, + /// The revision the adoption produced. + pub revision: u64, +} + +/// The durable control record of one fenced namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamespaceFenceRecord { + pub namespace: NamespaceName, + pub role: Role, + /// Always a durable state whose role is `role`. + pub state: FenceState, + /// Starts at 1 and increases by one on every applied transition. Survives restart. + pub revision: u64, + pub operation_id: Uuid, + pub identity: NamespaceIdentity, + pub drain_policy: Option, + /// When the current drain was requested, if the record is draining. + pub drain_started_at_ms: Option, + pub frozen_boundary: Option, + /// The most recent validation result of the owning operation (target). + pub validation: Option, + pub legacy_blocks: LegacyBlocks, + pub created_at_ms: i64, + pub last_transition_at_ms: i64, + /// The command that produced the current revision. + pub last_command_id: Uuid, + pub written_by: ServerIdentity, + pub adoptions: Vec, +} + +impl NamespaceFenceRecord { + pub fn write_admission(&self) -> Admission { + self.state.write_admission() + } + + pub fn read_admission(&self) -> Admission { + self.state.read_admission() + } + + /// Values of the legacy `block_*` configuration fields while this record is in force: the + /// fence state mirrored for an older binary, or the pre-fence values once the operation + /// has released the namespace. + pub fn legacy_mirror(&self) -> LegacyBlocks { + match self.state { + FenceState::Released | FenceState::TargetWritable => self.legacy_blocks.clone(), + state => LegacyBlocks { + block_reads: !state.read_admission().is_open(), + block_writes: !state.write_admission().is_open(), + block_reason: Some(format!( + "namespace fence: {state} (operation {})", + self.operation_id + )), + }, + } + } + + pub fn encode(&self) -> Vec { + codec::record_to_proto(self).encode_to_vec() + } + + /// Decode a stored record. `format_version` and `revision` are the columns stored beside + /// the payload; both must agree with it. + pub fn decode( + format_version: u32, + revision: u64, + bytes: &[u8], + ) -> Result { + check_format_version(format_version)?; + let msg = proto::FenceRecord::decode(bytes)?; + let record = codec::record_from_proto(msg)?; + if record.revision != revision { + return Err(FenceDecodeError::Invalid( + "revision column disagrees with payload", + )); + } + Ok(record) + } +} + +/// The durable result of one applied command, keyed by `(namespace, operation_id, command_id)`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandReceipt { + pub namespace: NamespaceName, + pub operation_id: Uuid, + pub command_id: Uuid, + pub command: CommandKind, + pub fingerprint: Fingerprint, + /// `APPLIED`, `ALREADY_APPLIED` or `DRAINING`. Errors are never stored. + pub outcome: FenceOutcome, + pub revision_before: u64, + pub revision_after: u64, + pub state_after: FenceState, + pub applied_at_ms: i64, + pub instance_id: Uuid, + pub adoption: Option, +} + +impl CommandReceipt { + /// Whether the receipt holds the command's final answer, as opposed to a drain that is + /// still to be completed. + pub fn is_final(&self) -> bool { + self.outcome != FenceOutcome::Draining + } + + pub fn encode(&self) -> Vec { + codec::receipt_to_proto(self).encode_to_vec() + } + + pub fn decode(format_version: u32, bytes: &[u8]) -> Result { + check_format_version(format_version)?; + let msg = proto::CommandReceipt::decode(bytes)?; + codec::receipt_from_proto(msg) + } +} + +/// Contents of the per-namespace marker file: a copy of the last committed record. Written +/// after each metastore commit, and before the metastore transaction of +/// `CreateTargetQuarantined`, so recovery can tell a fenced namespace from a legacy one and +/// detect a metastore that went backwards (section 5.6). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FenceMarker { + pub record: NamespaceFenceRecord, +} + +impl FenceMarker { + pub fn for_record(record: &NamespaceFenceRecord) -> Self { + Self { + record: record.clone(), + } + } + + pub fn encode(&self) -> Vec { + proto::FenceMarker { + format_version: FENCE_FORMAT_VERSION, + record: Some(codec::record_to_proto(&self.record)), + } + .encode_to_vec() + } + + pub fn decode(bytes: &[u8]) -> Result { + let msg = proto::FenceMarker::decode(bytes)?; + check_format_version(msg.format_version)?; + let record = msg + .record + .ok_or(FenceDecodeError::Invalid("marker without record"))?; + Ok(Self { + record: codec::record_from_proto(record)?, + }) + } +} + +/// Why stored fence state could not be read. Every variant means the namespace is +/// `UNKNOWN_UNAVAILABLE`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FenceDecodeError { + #[error("unsupported fence format version {0}")] + UnsupportedFormatVersion(u32), + #[error("undecodable fence payload: {0}")] + Undecodable(String), + #[error("invalid fence payload: {0}")] + Invalid(&'static str), +} + +impl From for FenceDecodeError { + fn from(e: prost::DecodeError) -> Self { + FenceDecodeError::Undecodable(e.to_string()) + } +} + +fn check_format_version(v: u32) -> Result<(), FenceDecodeError> { + if v != FENCE_FORMAT_VERSION { + return Err(FenceDecodeError::UnsupportedFormatVersion(v)); + } + Ok(()) +} + +/// Conversions between the domain types and their protobuf encoding. +pub(super) mod codec { + use super::*; + use crate::namespace::fence::command::OnDeadline; + + type R = Result; + + pub fn parse_uuid(s: &str) -> R { + // Only the canonical hyphenated form is ever written. + let id = Uuid::parse_str(s).map_err(|_| FenceDecodeError::Invalid("malformed id"))?; + if id.hyphenated().to_string() != s { + return Err(FenceDecodeError::Invalid("non-canonical id")); + } + Ok(id) + } + + fn parse_opt_uuid(s: Option<&str>) -> R> { + s.map(parse_uuid).transpose() + } + + fn namespace(s: String) -> R { + NamespaceName::from_string(s).map_err(|_| FenceDecodeError::Invalid("invalid namespace")) + } + + pub fn role_to_proto(role: Role) -> proto::FenceRole { + match role { + Role::Source => proto::FenceRole::Source, + Role::Target => proto::FenceRole::Target, + } + } + + pub fn role_from_proto(v: i32) -> R { + match proto::FenceRole::try_from(v) { + Ok(proto::FenceRole::Source) => Ok(Role::Source), + Ok(proto::FenceRole::Target) => Ok(Role::Target), + _ => Err(FenceDecodeError::Invalid("unknown role")), + } + } + + pub fn state_to_proto(state: FenceState) -> proto::FenceState { + use proto::FenceState as P; + match state { + FenceState::Unfenced => P::Unfenced, + FenceState::Absent => P::Absent, + FenceState::SourceDraining => P::SourceDraining, + FenceState::SourceWriteFenced => P::SourceWriteFenced, + FenceState::SourceReadDraining => P::SourceReadDraining, + FenceState::SourceReadFenced => P::SourceReadFenced, + FenceState::Released => P::Released, + FenceState::TargetQuarantined => P::TargetQuarantined, + FenceState::TargetImportDraining => P::TargetImportDraining, + FenceState::TargetValidating => P::TargetValidating, + FenceState::TargetWriteFenced => P::TargetWriteFenced, + FenceState::TargetWritable => P::TargetWritable, + FenceState::TargetAborted => P::TargetAborted, + FenceState::UnknownUnavailable => P::UnknownUnavailable, + } + } + + pub fn state_from_proto(v: i32) -> R { + use proto::FenceState as P; + let p = P::try_from(v).map_err(|_| FenceDecodeError::Invalid("unknown state"))?; + Ok(match p { + P::Unspecified => return Err(FenceDecodeError::Invalid("unspecified state")), + P::Unfenced => FenceState::Unfenced, + P::Absent => FenceState::Absent, + P::SourceDraining => FenceState::SourceDraining, + P::SourceWriteFenced => FenceState::SourceWriteFenced, + P::SourceReadDraining => FenceState::SourceReadDraining, + P::SourceReadFenced => FenceState::SourceReadFenced, + P::Released => FenceState::Released, + P::TargetQuarantined => FenceState::TargetQuarantined, + P::TargetImportDraining => FenceState::TargetImportDraining, + P::TargetValidating => FenceState::TargetValidating, + P::TargetWriteFenced => FenceState::TargetWriteFenced, + P::TargetWritable => FenceState::TargetWritable, + P::TargetAborted => FenceState::TargetAborted, + P::UnknownUnavailable => FenceState::UnknownUnavailable, + }) + } + + /// A state stored in a record or marker: durable, and of the stated role. + pub fn durable_state_from_proto(v: i32, role: Role) -> R { + let state = state_from_proto(v)?; + match state.role() { + Some(r) if r == role => Ok(state), + Some(_) => Err(FenceDecodeError::Invalid("state does not match role")), + None => Err(FenceDecodeError::Invalid("state is not durable")), + } + } + + pub fn command_kind_to_proto(kind: CommandKind) -> proto::CommandKind { + use proto::CommandKind as P; + match kind { + CommandKind::AcquireSourceWriteFence => P::AcquireSourceWriteFence, + CommandKind::SetSourceReadFence => P::SetSourceReadFence, + CommandKind::ClearSourceReadFence => P::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence => P::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined => P::CreateTargetQuarantined, + CommandKind::SealTargetImport => P::SealTargetImport, + CommandKind::RecordTargetValidation => P::RecordTargetValidation, + CommandKind::PublishTargetReadableWriteFenced => P::PublishTargetReadableWriteFenced, + CommandKind::EnableTargetWrites => P::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget => P::AbortQuarantinedTarget, + CommandKind::AdoptFence => P::AdoptFence, + } + } + + fn command_kind_from_proto(v: i32) -> R { + use proto::CommandKind as P; + let p = P::try_from(v).map_err(|_| FenceDecodeError::Invalid("unknown command"))?; + Ok(match p { + P::Unspecified => return Err(FenceDecodeError::Invalid("unspecified command")), + P::AcquireSourceWriteFence => CommandKind::AcquireSourceWriteFence, + P::SetSourceReadFence => CommandKind::SetSourceReadFence, + P::ClearSourceReadFence => CommandKind::ClearSourceReadFence, + P::ReleaseSourceWriteFence => CommandKind::ReleaseSourceWriteFence, + P::CreateTargetQuarantined => CommandKind::CreateTargetQuarantined, + P::SealTargetImport => CommandKind::SealTargetImport, + P::RecordTargetValidation => CommandKind::RecordTargetValidation, + P::PublishTargetReadableWriteFenced => CommandKind::PublishTargetReadableWriteFenced, + P::EnableTargetWrites => CommandKind::EnableTargetWrites, + P::AbortQuarantinedTarget => CommandKind::AbortQuarantinedTarget, + P::AdoptFence => CommandKind::AdoptFence, + }) + } + + fn outcome_to_proto(outcome: FenceOutcome) -> proto::ReceiptOutcome { + match outcome { + FenceOutcome::Applied => proto::ReceiptOutcome::Applied, + FenceOutcome::AlreadyApplied => proto::ReceiptOutcome::AlreadyApplied, + FenceOutcome::Draining => proto::ReceiptOutcome::Draining, + other => unreachable!("receipts never store {other}"), + } + } + + fn outcome_from_proto(v: i32) -> R { + match proto::ReceiptOutcome::try_from(v) { + Ok(proto::ReceiptOutcome::Applied) => Ok(FenceOutcome::Applied), + Ok(proto::ReceiptOutcome::AlreadyApplied) => Ok(FenceOutcome::AlreadyApplied), + Ok(proto::ReceiptOutcome::Draining) => Ok(FenceOutcome::Draining), + _ => Err(FenceDecodeError::Invalid("unknown receipt outcome")), + } + } + + pub fn drain_policy_to_proto(p: &DrainPolicy) -> proto::DrainPolicy { + proto::DrainPolicy { + deadline_ms: p.deadline_ms, + on_deadline: match p.on_deadline { + OnDeadline::Fail => proto::OnDeadline::Fail, + OnDeadline::ForceRollback => proto::OnDeadline::ForceRollback, + } as i32, + } + } + + fn drain_policy_from_proto(p: proto::DrainPolicy) -> R { + let on_deadline = match proto::OnDeadline::try_from(p.on_deadline) { + Ok(proto::OnDeadline::Fail) => OnDeadline::Fail, + Ok(proto::OnDeadline::ForceRollback) => OnDeadline::ForceRollback, + _ => return Err(FenceDecodeError::Invalid("unknown drain deadline policy")), + }; + Ok(DrainPolicy { + deadline_ms: p.deadline_ms, + on_deadline, + }) + } + + pub fn validation_result_to_proto(r: ValidationResult) -> proto::ValidationResult { + match r { + ValidationResult::Ok => proto::ValidationResult::Ok, + ValidationResult::Failed => proto::ValidationResult::Failed, + } + } + + fn validation_result_from_proto(v: i32) -> R { + match proto::ValidationResult::try_from(v) { + Ok(proto::ValidationResult::Ok) => Ok(ValidationResult::Ok), + Ok(proto::ValidationResult::Failed) => Ok(ValidationResult::Failed), + _ => Err(FenceDecodeError::Invalid("unknown validation result")), + } + } + + pub fn target_config_to_proto(c: &TargetConfig) -> proto::TargetConfig { + proto::TargetConfig { + max_db_size: c.max_db_size, + jwt_key: c.jwt_key.clone(), + txn_timeout_s: c.txn_timeout_s, + allow_attach: c.allow_attach, + durability_mode: c.durability_mode.clone(), + bottomless_db_id: c.bottomless_db_id.clone(), + } + } + + fn validation_to_proto(v: &ValidationRecord) -> proto::ValidationRecord { + proto::ValidationRecord { + operation_id: v.operation_id.to_string(), + command_id: v.command_id.to_string(), + result: validation_result_to_proto(v.result) as i32, + summary: v.summary.clone(), + snapshot: v.snapshot.map(|s| proto::ValidationSnapshot { + log_id: s.log_id.to_string(), + frame_no: s.frame_no, + page_count: s.page_count, + }), + recorded_at_ms: v.recorded_at_ms, + } + } + + fn validation_from_proto(v: proto::ValidationRecord) -> R { + Ok(ValidationRecord { + operation_id: parse_uuid(&v.operation_id)?, + command_id: parse_uuid(&v.command_id)?, + result: validation_result_from_proto(v.result)?, + summary: v.summary, + snapshot: v + .snapshot + .map(|s| { + Ok::<_, FenceDecodeError>(ValidationSnapshot { + log_id: parse_uuid(&s.log_id)?, + frame_no: s.frame_no, + page_count: s.page_count, + }) + }) + .transpose()?, + recorded_at_ms: v.recorded_at_ms, + }) + } + + fn adoption_to_proto(a: &Adoption) -> proto::Adoption { + proto::Adoption { + previous_operation_id: a.previous_operation_id.to_string(), + new_operation_id: a.new_operation_id.to_string(), + command_id: a.command_id.to_string(), + approvers: a.approvers.clone(), + incident_ref: a.incident_ref.clone(), + reason: a.reason.clone(), + at_ms: a.at_ms, + revision: a.revision, + } + } + + fn adoption_from_proto(a: proto::Adoption) -> R { + Ok(Adoption { + previous_operation_id: parse_uuid(&a.previous_operation_id)?, + new_operation_id: parse_uuid(&a.new_operation_id)?, + command_id: parse_uuid(&a.command_id)?, + approvers: a.approvers, + incident_ref: a.incident_ref, + reason: a.reason, + at_ms: a.at_ms, + revision: a.revision, + }) + } + + pub fn record_to_proto(r: &NamespaceFenceRecord) -> proto::FenceRecord { + proto::FenceRecord { + namespace: r.namespace.as_str().to_string(), + role: role_to_proto(r.role) as i32, + state: state_to_proto(r.state) as i32, + revision: r.revision, + operation_id: r.operation_id.to_string(), + log_id: r.identity.log_id.map(|id| id.to_string()), + target_incarnation_id: r.identity.target_incarnation_id.map(|id| id.to_string()), + drain_policy: r.drain_policy.as_ref().map(drain_policy_to_proto), + drain_started_at_ms: r.drain_started_at_ms, + frozen_boundary: r.frozen_boundary.map(|b| proto::FrozenBoundary { + log_id: b.log_id.to_string(), + frame_no: b.frame_no, + }), + validation: r.validation.as_ref().map(validation_to_proto), + legacy_blocks: Some(proto::LegacyBlocks { + block_reads: r.legacy_blocks.block_reads, + block_writes: r.legacy_blocks.block_writes, + block_reason: r.legacy_blocks.block_reason.clone(), + }), + created_at_ms: r.created_at_ms, + last_transition_at_ms: r.last_transition_at_ms, + last_command_id: r.last_command_id.to_string(), + written_by: Some(proto::ServerIdentity { + build: r.written_by.build.clone(), + instance_id: r.written_by.instance_id.to_string(), + }), + adoptions: r.adoptions.iter().map(adoption_to_proto).collect(), + } + } + + pub fn record_from_proto(m: proto::FenceRecord) -> R { + let role = role_from_proto(m.role)?; + let state = durable_state_from_proto(m.state, role)?; + if m.revision == 0 { + return Err(FenceDecodeError::Invalid("record revision is zero")); + } + let legacy = m + .legacy_blocks + .ok_or(FenceDecodeError::Invalid("missing legacy blocks"))?; + let written_by = m + .written_by + .ok_or(FenceDecodeError::Invalid("missing server identity"))?; + let record = NamespaceFenceRecord { + namespace: namespace(m.namespace)?, + role, + state, + revision: m.revision, + operation_id: parse_uuid(&m.operation_id)?, + identity: NamespaceIdentity { + log_id: parse_opt_uuid(m.log_id.as_deref())?, + target_incarnation_id: parse_opt_uuid(m.target_incarnation_id.as_deref())?, + }, + drain_policy: m.drain_policy.map(drain_policy_from_proto).transpose()?, + drain_started_at_ms: m.drain_started_at_ms, + frozen_boundary: m + .frozen_boundary + .map(|b| { + Ok::<_, FenceDecodeError>(FrozenBoundary { + log_id: parse_uuid(&b.log_id)?, + frame_no: b.frame_no, + }) + }) + .transpose()?, + validation: m.validation.map(validation_from_proto).transpose()?, + legacy_blocks: LegacyBlocks { + block_reads: legacy.block_reads, + block_writes: legacy.block_writes, + block_reason: legacy.block_reason, + }, + created_at_ms: m.created_at_ms, + last_transition_at_ms: m.last_transition_at_ms, + last_command_id: parse_uuid(&m.last_command_id)?, + written_by: ServerIdentity { + build: written_by.build, + instance_id: parse_uuid(&written_by.instance_id)?, + }, + adoptions: m + .adoptions + .into_iter() + .map(adoption_from_proto) + .collect::>()?, + }; + check_record_invariants(&record)?; + Ok(record) + } + + /// Facts every record written by `apply` satisfies. A record that breaks one was not + /// written by this server and is not trusted. + fn check_record_invariants(r: &NamespaceFenceRecord) -> R<()> { + use FenceState::*; + let frozen_required = matches!( + r.state, + SourceWriteFenced | SourceReadDraining | SourceReadFenced + ); + if frozen_required && r.frozen_boundary.is_none() { + return Err(FenceDecodeError::Invalid( + "fenced source without frozen boundary", + )); + } + if r.role == Role::Source && r.identity.log_id.is_none() { + return Err(FenceDecodeError::Invalid( + "source without namespace identity", + )); + } + if r.role == Role::Target && r.identity.target_incarnation_id.is_none() { + return Err(FenceDecodeError::Invalid("target without incarnation id")); + } + if matches!(r.state, TargetWriteFenced | TargetWritable) + && !r + .validation + .as_ref() + .is_some_and(|v| v.result == ValidationResult::Ok) + { + return Err(FenceDecodeError::Invalid( + "published target without validation", + )); + } + Ok(()) + } + + pub fn receipt_to_proto(r: &CommandReceipt) -> proto::CommandReceipt { + proto::CommandReceipt { + namespace: r.namespace.as_str().to_string(), + operation_id: r.operation_id.to_string(), + command_id: r.command_id.to_string(), + command: command_kind_to_proto(r.command) as i32, + fingerprint: r.fingerprint.as_bytes().to_vec(), + outcome: outcome_to_proto(r.outcome) as i32, + revision_before: r.revision_before, + revision_after: r.revision_after, + state_after: state_to_proto(r.state_after) as i32, + applied_at_ms: r.applied_at_ms, + instance_id: r.instance_id.to_string(), + adoption: r.adoption.as_ref().map(adoption_to_proto), + } + } + + pub fn receipt_from_proto(m: proto::CommandReceipt) -> R { + let fingerprint: [u8; 32] = m + .fingerprint + .as_slice() + .try_into() + .map_err(|_| FenceDecodeError::Invalid("fingerprint is not 32 bytes"))?; + let state_after = state_from_proto(m.state_after)?; + if !state_after.is_durable() { + return Err(FenceDecodeError::Invalid("receipt state is not durable")); + } + if m.revision_after < m.revision_before { + return Err(FenceDecodeError::Invalid("receipt revision went backwards")); + } + Ok(CommandReceipt { + namespace: namespace(m.namespace)?, + operation_id: parse_uuid(&m.operation_id)?, + command_id: parse_uuid(&m.command_id)?, + command: command_kind_from_proto(m.command)?, + fingerprint: Fingerprint(fingerprint), + outcome: outcome_from_proto(m.outcome)?, + revision_before: m.revision_before, + revision_after: m.revision_after, + state_after, + applied_at_ms: m.applied_at_ms, + instance_id: parse_uuid(&m.instance_id)?, + adoption: m.adoption.map(adoption_from_proto).transpose()?, + }) + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + use crate::namespace::fence::command::OnDeadline; + + pub fn sample_record() -> NamespaceFenceRecord { + NamespaceFenceRecord { + namespace: NamespaceName::from("db1"), + role: Role::Source, + state: FenceState::SourceWriteFenced, + revision: 2, + operation_id: Uuid::from_u128(1), + identity: NamespaceIdentity { + log_id: Some(Uuid::from_u128(10)), + target_incarnation_id: None, + }, + drain_policy: Some(DrainPolicy { + deadline_ms: 5000, + on_deadline: OnDeadline::ForceRollback, + }), + drain_started_at_ms: Some(100), + frozen_boundary: Some(FrozenBoundary { + log_id: Uuid::from_u128(10), + frame_no: 1234, + }), + validation: None, + legacy_blocks: LegacyBlocks { + block_reads: false, + block_writes: true, + block_reason: Some("maintenance".into()), + }, + created_at_ms: 100, + last_transition_at_ms: 200, + last_command_id: Uuid::from_u128(2), + written_by: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(99), + }, + adoptions: vec![Adoption { + previous_operation_id: Uuid::from_u128(5), + new_operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(6), + approvers: vec!["a".into(), "b".into()], + incident_ref: "inc".into(), + reason: "lost control plane".into(), + at_ms: 150, + revision: 2, + }], + } + } + + pub fn sample_receipt() -> CommandReceipt { + CommandReceipt { + namespace: NamespaceName::from("db1"), + operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(2), + command: CommandKind::AcquireSourceWriteFence, + fingerprint: Fingerprint([7; 32]), + outcome: FenceOutcome::Applied, + revision_before: 0, + revision_after: 2, + state_after: FenceState::SourceWriteFenced, + applied_at_ms: 200, + instance_id: Uuid::from_u128(99), + adoption: None, + } + } + + #[test] + fn record_round_trips() { + let record = sample_record(); + let bytes = record.encode(); + let decoded = NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 2, &bytes).unwrap(); + assert_eq!(decoded, record); + } + + #[test] + fn receipt_round_trips() { + let receipt = sample_receipt(); + let decoded = CommandReceipt::decode(FENCE_FORMAT_VERSION, &receipt.encode()).unwrap(); + assert_eq!(decoded, receipt); + } + + #[test] + fn marker_round_trips() { + let marker = FenceMarker::for_record(&sample_record()); + assert_eq!(FenceMarker::decode(&marker.encode()).unwrap(), marker); + } + + #[test] + fn unknown_format_version_is_rejected() { + let bytes = sample_record().encode(); + assert_eq!( + NamespaceFenceRecord::decode(2, 2, &bytes), + Err(FenceDecodeError::UnsupportedFormatVersion(2)) + ); + assert!(matches!( + CommandReceipt::decode(0, &sample_receipt().encode()), + Err(FenceDecodeError::UnsupportedFormatVersion(0)) + )); + let mut marker = proto::FenceMarker::decode( + FenceMarker::for_record(&sample_record()) + .encode() + .as_slice(), + ) + .unwrap(); + marker.format_version = 7; + assert_eq!( + FenceMarker::decode(&marker.encode_to_vec()), + Err(FenceDecodeError::UnsupportedFormatVersion(7)) + ); + } + + #[test] + fn garbage_is_rejected() { + assert!(matches!( + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 2, &[0xff, 0xff, 0xff]), + Err(FenceDecodeError::Undecodable(_)) + )); + // An empty payload decodes as an all-default message, which is not a valid record. + assert!(NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 0, &[]).is_err()); + assert!(CommandReceipt::decode(FENCE_FORMAT_VERSION, &[]).is_err()); + assert!(FenceMarker::decode(&[]).is_err()); + } + + #[test] + fn revision_column_must_match() { + let bytes = sample_record().encode(); + assert!(matches!( + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 3, &bytes), + Err(FenceDecodeError::Invalid(_)) + )); + } + + fn mutate( + f: impl FnOnce(&mut proto::FenceRecord), + ) -> Result { + let mut m = codec::record_to_proto(&sample_record()); + f(&mut m); + let revision = m.revision; + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, revision, &m.encode_to_vec()) + } + + #[test] + fn invalid_records_are_rejected() { + assert!(mutate(|m| m.state = 999).is_err(), "unknown state"); + assert!( + mutate(|m| m.state = proto::FenceState::Unfenced as i32).is_err(), + "non-durable" + ); + assert!( + mutate(|m| m.state = proto::FenceState::UnknownUnavailable as i32).is_err(), + "derived state stored" + ); + assert!( + mutate(|m| m.state = proto::FenceState::TargetWritable as i32).is_err(), + "state/role mismatch" + ); + assert!(mutate(|m| m.role = 0).is_err(), "unspecified role"); + assert!(mutate(|m| m.operation_id = "not-a-uuid".into()).is_err()); + assert!( + mutate(|m| m.operation_id = m.operation_id.replace('-', "")).is_err(), + "non-canonical id" + ); + assert!(mutate(|m| m.namespace = String::new()).is_err()); + assert!(mutate(|m| m.legacy_blocks = None).is_err()); + assert!(mutate(|m| m.written_by = None).is_err()); + assert!( + mutate(|m| m.frozen_boundary = None).is_err(), + "fenced without boundary" + ); + assert!( + mutate(|m| m.log_id = None).is_err(), + "source without identity" + ); + assert!( + mutate(|m| { + m.revision = 0; + }) + .is_err(), + "revision zero" + ); + assert!( + mutate(|m| m.drain_policy.as_mut().unwrap().on_deadline = 0).is_err(), + "unspecified drain policy" + ); + } + + #[test] + fn invalid_receipts_are_rejected() { + let enc = |f: &dyn Fn(&mut proto::CommandReceipt)| { + let mut m = codec::receipt_to_proto(&sample_receipt()); + f(&mut m); + CommandReceipt::decode(FENCE_FORMAT_VERSION, &m.encode_to_vec()) + }; + assert!(enc(&|m| m.fingerprint = vec![1; 31]).is_err()); + assert!(enc(&|m| m.outcome = 0).is_err()); + assert!(enc(&|m| m.command = 0).is_err()); + assert!(enc(&|m| m.state_after = proto::FenceState::Unfenced as i32).is_err()); + assert!( + enc(&|m| m.revision_before = 5).is_err(), + "revision went backwards" + ); + } + + #[test] + fn legacy_mirror_follows_state() { + let mut record = sample_record(); + let m = record.legacy_mirror(); + assert!(!m.block_reads); + assert!(m.block_writes); + assert!(m.block_reason.unwrap().contains("SOURCE_WRITE_FENCED")); + + record.state = FenceState::SourceReadFenced; + let m = record.legacy_mirror(); + assert!(m.block_reads && m.block_writes); + + record.state = FenceState::Released; + assert_eq!(record.legacy_mirror(), record.legacy_blocks); + + record.role = Role::Target; + record.state = FenceState::TargetQuarantined; + let m = record.legacy_mirror(); + assert!(m.block_reads && m.block_writes); + record.state = FenceState::TargetWriteFenced; + let m = record.legacy_mirror(); + assert!(!m.block_reads && m.block_writes); + } +} diff --git a/libsql-server/src/namespace/fence/state.rs b/libsql-server/src/namespace/fence/state.rs new file mode 100644 index 0000000000..53a681775c --- /dev/null +++ b/libsql-server/src/namespace/fence/state.rs @@ -0,0 +1,403 @@ +//! Roles, states, operation classes and the permission matrix of `docs/NAMESPACE_FENCE.md` +//! sections 3 and 7.3. + +use std::fmt; +use std::str::FromStr; + +use super::outcome::FenceOutcome; + +/// Which side of a move a fence record belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Role { + Source, + Target, +} + +impl Role { + pub const fn as_str(self) -> &'static str { + match self { + Role::Source => "SOURCE", + Role::Target => "TARGET", + } + } +} + +impl fmt::Display for Role { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The state of a namespace as the fence sees it. +/// +/// `Unfenced` and `Absent` describe a namespace with no record (an ordinary namespace, or no +/// namespace at all). `UnknownUnavailable` is derived when the server cannot establish the +/// control state. None of those three is ever stored in a record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceState { + Unfenced, + Absent, + SourceDraining, + SourceWriteFenced, + SourceReadDraining, + SourceReadFenced, + Released, + TargetQuarantined, + TargetImportDraining, + TargetValidating, + TargetWriteFenced, + TargetWritable, + TargetAborted, + UnknownUnavailable, +} + +impl FenceState { + pub const ALL: [FenceState; 14] = [ + FenceState::Unfenced, + FenceState::Absent, + FenceState::SourceDraining, + FenceState::SourceWriteFenced, + FenceState::SourceReadDraining, + FenceState::SourceReadFenced, + FenceState::Released, + FenceState::TargetQuarantined, + FenceState::TargetImportDraining, + FenceState::TargetValidating, + FenceState::TargetWriteFenced, + FenceState::TargetWritable, + FenceState::TargetAborted, + FenceState::UnknownUnavailable, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + FenceState::Unfenced => "UNFENCED", + FenceState::Absent => "ABSENT", + FenceState::SourceDraining => "SOURCE_DRAINING", + FenceState::SourceWriteFenced => "SOURCE_WRITE_FENCED", + FenceState::SourceReadDraining => "SOURCE_READ_DRAINING", + FenceState::SourceReadFenced => "SOURCE_READ_FENCED", + FenceState::Released => "RELEASED", + FenceState::TargetQuarantined => "TARGET_QUARANTINED", + FenceState::TargetImportDraining => "TARGET_IMPORT_DRAINING", + FenceState::TargetValidating => "TARGET_VALIDATING", + FenceState::TargetWriteFenced => "TARGET_WRITE_FENCED", + FenceState::TargetWritable => "TARGET_WRITABLE", + FenceState::TargetAborted => "TARGET_ABORTED", + FenceState::UnknownUnavailable => "UNKNOWN_UNAVAILABLE", + } + } + + /// The role a record in this state has, if the state belongs to one. + pub const fn role(self) -> Option { + match self { + FenceState::SourceDraining + | FenceState::SourceWriteFenced + | FenceState::SourceReadDraining + | FenceState::SourceReadFenced + | FenceState::Released => Some(Role::Source), + FenceState::TargetQuarantined + | FenceState::TargetImportDraining + | FenceState::TargetValidating + | FenceState::TargetWriteFenced + | FenceState::TargetWritable + | FenceState::TargetAborted => Some(Role::Target), + FenceState::Unfenced | FenceState::Absent | FenceState::UnknownUnavailable => None, + } + } + + /// Whether this state can be stored in a fence record. + pub const fn is_durable(self) -> bool { + self.role().is_some() + } + + /// Whether the operation that owns a record in this state has finished with it. + pub const fn is_operation_finished(self) -> bool { + matches!( + self, + FenceState::Released | FenceState::TargetWritable | FenceState::TargetAborted + ) + } + + /// Whether a record in this state counts as an active fence (section 4.4, + /// `active_fences`): anything except an ordinary namespace, a released source or a + /// published target. + pub const fn is_active(self) -> bool { + !matches!( + self, + FenceState::Unfenced + | FenceState::Absent + | FenceState::Released + | FenceState::TargetWritable + ) + } + + /// A state in which the server is waiting for work admitted earlier to end. + pub const fn is_draining(self) -> bool { + matches!( + self, + FenceState::SourceDraining + | FenceState::SourceReadDraining + | FenceState::TargetImportDraining + ) + } + + /// The permission-matrix decision for work of `class` (section 3.3). + /// + /// For the capability classes this decides only whether the state admits capability work + /// at all; whether a particular capability is valid is the controller's decision. + pub fn permits(self, class: OperationClass) -> Result<(), FenceOutcome> { + use FenceState::*; + use OperationClass::*; + + match class { + Maintenance | Observability => return Ok(()), + _ => (), + } + + if self == UnknownUnavailable { + return Err(FenceOutcome::FenceStateUnavailable); + } + + match class { + NormalRead | Stream => match self { + Unfenced | Absent | Released | SourceDraining | SourceWriteFenced + | TargetWriteFenced | TargetWritable => Ok(()), + SourceReadDraining | SourceReadFenced => Err(FenceOutcome::MigrationReadFenced), + TargetQuarantined | TargetImportDraining | TargetValidating | TargetAborted => { + Err(FenceOutcome::MigrationTargetQuarantined) + } + UnknownUnavailable => unreachable!(), + }, + NormalWrite | Vacuum | Lifecycle => match self { + Unfenced | Absent | Released | TargetWritable => Ok(()), + SourceDraining | SourceWriteFenced | SourceReadDraining | SourceReadFenced + | TargetWriteFenced => Err(FenceOutcome::MigrationWriteFenced), + TargetQuarantined | TargetImportDraining | TargetValidating | TargetAborted => { + Err(FenceOutcome::MigrationTargetQuarantined) + } + UnknownUnavailable => unreachable!(), + }, + CapabilityImport => match self { + TargetQuarantined => Ok(()), + // Existing import writers may finish while draining, but no new one starts: + // that distinction is the controller's, which tracks issued capabilities. + TargetImportDraining => Ok(()), + _ => Err(FenceOutcome::OperationCapabilityRequired), + }, + CapabilityValidate => match self { + TargetValidating | TargetWriteFenced => Ok(()), + _ => Err(FenceOutcome::OperationCapabilityRequired), + }, + Maintenance | Observability => unreachable!(), + } + } + + /// Whether normal write admission is open in this state. + pub fn write_admission(self) -> Admission { + Admission::from(self.permits(OperationClass::NormalWrite)) + } + + /// Whether normal read admission is open in this state. + pub fn read_admission(self) -> Admission { + Admission::from(self.permits(OperationClass::NormalRead)) + } +} + +impl fmt::Display for FenceState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown fence state `{0}`")] +pub struct UnknownFenceState(pub String); + +impl FromStr for FenceState { + type Err = UnknownFenceState; + + fn from_str(s: &str) -> Result { + FenceState::ALL + .iter() + .copied() + .find(|state| state.as_str() == s) + .ok_or_else(|| UnknownFenceState(s.to_string())) + } +} + +/// Whether an admission path is open, and if it is closed, the code a denial carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Admission { + Open, + Closed(FenceOutcome), +} + +impl Admission { + pub fn is_open(self) -> bool { + matches!(self, Admission::Open) + } + + pub fn as_str(self) -> &'static str { + match self { + Admission::Open => "open", + Admission::Closed(_) => "closed", + } + } +} + +impl From> for Admission { + fn from(value: Result<(), FenceOutcome>) -> Self { + match value { + Ok(()) => Admission::Open, + Err(code) => Admission::Closed(code), + } + } +} + +/// The class of a piece of work, which the permission matrix is keyed by (section 7.3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OperationClass { + /// Any logical write that is not operation-owned: SQL over every protocol, the admin + /// shell, schema migration, dump load outside an import capability. + NormalWrite, + /// Work that cannot change logical contents: `TRUNCATE` checkpoints, the storage monitor, + /// bottomless WAL upload, the replication logger's own connection. + Maintenance, + /// `VACUUM`. It takes a write transaction and produces replicated frames, so it is not + /// maintenance and is skipped wherever normal writes are denied. + Vacuum, + /// Generic lifecycle and configuration: config mutation, delete, reset, fork, create over + /// an existing record, restore, dump load, shared-schema linking. + Lifecycle, + /// Writes of an import session holding a `MigrationCapability`. + CapabilityImport, + /// Read-only validation through a `MigrationCapability`. + CapabilityValidate, + /// SQL programs, Hrana cursors, `/beta/listen`, ATTACH of the namespace. + NormalRead, + /// `/dump` and replication streams (`hello`, `log_entries`, `batch_log_entries`, + /// `snapshot`). + Stream, + /// Stats, jobs and metrics. Never a read lease. + Observability, +} + +impl OperationClass { + pub const ALL: [OperationClass; 9] = [ + OperationClass::NormalWrite, + OperationClass::Maintenance, + OperationClass::Vacuum, + OperationClass::Lifecycle, + OperationClass::CapabilityImport, + OperationClass::CapabilityValidate, + OperationClass::NormalRead, + OperationClass::Stream, + OperationClass::Observability, + ]; +} + +#[cfg(test)] +mod tests { + use super::*; + + use FenceOutcome as O; + use FenceState as S; + use OperationClass as C; + + #[test] + fn state_names_round_trip() { + for state in FenceState::ALL { + assert_eq!(state.as_str().parse::().unwrap(), state); + } + assert!("source_draining".parse::().is_err()); + assert!("".parse::().is_err()); + } + + #[test] + fn durable_states_have_a_role() { + for state in FenceState::ALL { + let expected = !matches!(state, S::Unfenced | S::Absent | S::UnknownUnavailable); + assert_eq!(state.is_durable(), expected, "{state}"); + } + } + + /// The whole permission matrix of section 3.3, row by row. + #[test] + fn permission_matrix() { + let allow = Ok(()); + let wf = Err(O::MigrationWriteFenced); + let rf = Err(O::MigrationReadFenced); + let tq = Err(O::MigrationTargetQuarantined); + let un = Err(O::FenceStateUnavailable); + let cap = Err(O::OperationCapabilityRequired); + + // state: (read, stream, write, lifecycle, import, validate) + let rows = [ + (S::Unfenced, [allow, allow, allow, allow, cap, cap]), + (S::Released, [allow, allow, allow, allow, cap, cap]), + (S::SourceDraining, [allow, allow, wf, wf, cap, cap]), + (S::SourceWriteFenced, [allow, allow, wf, wf, cap, cap]), + (S::SourceReadDraining, [rf, rf, wf, wf, cap, cap]), + (S::SourceReadFenced, [rf, rf, wf, wf, cap, cap]), + (S::TargetQuarantined, [tq, tq, tq, tq, allow, cap]), + (S::TargetImportDraining, [tq, tq, tq, tq, allow, cap]), + (S::TargetValidating, [tq, tq, tq, tq, cap, allow]), + (S::TargetWriteFenced, [allow, allow, wf, wf, cap, allow]), + (S::TargetWritable, [allow, allow, allow, allow, cap, cap]), + (S::TargetAborted, [tq, tq, tq, tq, cap, cap]), + (S::UnknownUnavailable, [un, un, un, un, un, un]), + ]; + + for (state, expected) in rows { + let classes = [ + C::NormalRead, + C::Stream, + C::NormalWrite, + C::Lifecycle, + C::CapabilityImport, + C::CapabilityValidate, + ]; + for (class, expected) in classes.into_iter().zip(expected) { + assert_eq!(state.permits(class), expected, "{state} {class:?}"); + } + // Vacuum follows the normal write column. + assert_eq!( + state.permits(C::Vacuum), + state.permits(C::NormalWrite), + "{state}" + ); + // Maintenance and observability continue in every state. + assert_eq!(state.permits(C::Maintenance), Ok(()), "{state}"); + assert_eq!(state.permits(C::Observability), Ok(()), "{state}"); + } + } + + #[test] + fn denials_are_data_plane_codes() { + for state in FenceState::ALL { + for class in OperationClass::ALL { + if let Err(code) = state.permits(class) { + assert!(code.is_error(), "{state} {class:?} {code}"); + } + } + } + } + + #[test] + fn active_and_finished() { + assert!(!S::Unfenced.is_active()); + assert!(!S::Released.is_active()); + assert!(!S::TargetWritable.is_active()); + assert!(S::TargetAborted.is_active()); + assert!(S::UnknownUnavailable.is_active()); + assert!(S::SourceDraining.is_active()); + + for state in FenceState::ALL { + assert_eq!( + state.is_operation_finished(), + matches!(state, S::Released | S::TargetWritable | S::TargetAborted) + ); + } + } +} diff --git a/libsql-server/src/namespace/fence/store.rs b/libsql-server/src/namespace/fence/store.rs new file mode 100644 index 0000000000..bb7a92a20f --- /dev/null +++ b/libsql-server/src/namespace/fence/store.rs @@ -0,0 +1,990 @@ +//! Durable fence state: the metastore tables, the fence compare-and-swap, and the +//! per-namespace marker file (`docs/NAMESPACE_FENCE.md` sections 5.1 and 5.4 to 5.6). +//! +//! Everything here runs on a metastore connection, inside a transaction the caller opened with +//! `BEGIN IMMEDIATE`, so a fence transition, an ordinary config write and a delete of the same +//! namespace are serialised by SQLite's write lock whichever connection they use. The functions +//! only read and write rows and files: what a command does is decided by +//! [`transition::apply`](super::transition::apply), and when the result is published is the +//! caller's decision, made only after the transaction has committed. +//! +//! Reading is strict. A row with an unknown format version, an undecodable payload, or a +//! revision column that disagrees with its payload, and a marker that says more than the +//! metastore does, all read as [`StoredFence::Unavailable`]: the namespace is +//! `UNKNOWN_UNAVAILABLE` and every gate that consults it stays closed. + +use std::fs::{self, File}; +use std::io::{self, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use prost::Message as _; +use rusqlite::{params, OptionalExtension}; +use uuid::Uuid; + +use crate::connection::config::{DatabaseConfig, DurabilityMode}; +use crate::namespace::NamespaceName; +use crate::LIBSQL_PAGE_SIZE; +use libsql_replication::rpc::metadata; + +use super::command::TargetConfig; +use super::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::record::{ + CommandReceipt, FenceDecodeError, FenceMarker, LegacyBlocks, NamespaceFenceRecord, + FENCE_FORMAT_VERSION, +}; +use super::state::{FenceState, OperationClass}; +use super::transition::CurrentFence; + +/// Name of the per-namespace marker file, inside the namespace's directory. +pub const MARKER_FILE_NAME: &str = ".fence"; + +/// How long receipts of finished operations are kept by default (section 5.5). +pub const DEFAULT_RECEIPT_RETENTION: Duration = Duration::from_secs(30 * 24 * 60 * 60); + +const CREATE_FENCES_TABLE: &str = " + CREATE TABLE IF NOT EXISTS namespace_fences ( + namespace TEXT NOT NULL PRIMARY KEY, + format_version INTEGER NOT NULL, + revision INTEGER NOT NULL, + record BLOB NOT NULL, + FOREIGN KEY (namespace) REFERENCES namespace_configs (namespace) + ON DELETE RESTRICT ON UPDATE RESTRICT + )"; + +const CREATE_RECEIPTS_TABLE: &str = " + CREATE TABLE IF NOT EXISTS namespace_fence_receipts ( + namespace TEXT NOT NULL, + operation_id TEXT NOT NULL, + command_id TEXT NOT NULL, + format_version INTEGER NOT NULL, + revision_after INTEGER NOT NULL, + applied_at INTEGER NOT NULL, + receipt BLOB NOT NULL, + PRIMARY KEY (namespace, operation_id, command_id) + )"; + +/// Errors of the persistence layer. A fence outcome is a result the caller answers with; the +/// others are faults of the metastore or the filesystem. +#[derive(Debug, thiserror::Error)] +pub enum FenceStoreError { + #[error(transparent)] + Fence(#[from] FenceError), + #[error("metastore error: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("fence marker I/O error: {0}")] + Io(#[from] io::Error), +} + +/// Create the fence tables. Called when the fence is enabled; the tables are additive, and +/// the metastore of a server that never enabled the fence does not have them. +pub fn create_tables(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + conn.execute(CREATE_FENCES_TABLE, ())?; + conn.execute(CREATE_RECEIPTS_TABLE, ())?; + Ok(()) +} + +/// Whether this metastore has ever held fence state. Once it has, fences are loaded and +/// enforced whether or not the fence is enabled (section 13.1). +pub fn tables_exist(conn: &rusqlite::Connection) -> rusqlite::Result { + let count: i64 = conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' + AND name IN ('namespace_fences', 'namespace_fence_receipts')", + (), + |row| row.get(0), + )?; + Ok(count > 0) +} + +/// What the store established about one namespace's fence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoredFence { + /// No fence record and no marker. `namespace_exists` is whether the namespace has a config + /// row: `UNFENCED` when it does, `ABSENT` when it does not. + None { + namespace_exists: bool, + }, + Record(NamespaceFenceRecord), + /// The control state cannot be established: `UNKNOWN_UNAVAILABLE`. + Unavailable { + detail: FenceDetail, + /// What was wrong, for the operator log and `InspectFence`. + reason: String, + /// The record the marker file holds, when it has a readable one. + marker: Option, + }, +} + +impl StoredFence { + pub fn as_current(&self) -> CurrentFence<'_> { + match self { + StoredFence::None { namespace_exists } => CurrentFence::None { + namespace_exists: *namespace_exists, + }, + StoredFence::Record(r) => CurrentFence::Record(r), + StoredFence::Unavailable { detail, marker, .. } => CurrentFence::Unavailable { + detail: *detail, + marker: marker.as_ref(), + }, + } + } + + pub fn state(&self) -> FenceState { + self.as_current().state() + } + + pub fn revision(&self) -> u64 { + self.as_current().revision() + } + + pub fn record(&self) -> Option<&NamespaceFenceRecord> { + match self { + StoredFence::Record(r) => Some(r), + _ => None, + } + } + + /// The permission-matrix decision for work of `class`, as an error a caller can return. + pub fn permits(&self, class: OperationClass) -> Result<(), FenceError> { + self.state().permits(class).map_err(|outcome| { + let err = FenceError::new(outcome, self.denial_message(class)); + match self { + StoredFence::Unavailable { detail, .. } => err.with_detail(*detail), + _ => err, + } + }) + } + + fn denial_message(&self, class: OperationClass) -> String { + match self { + StoredFence::Record(r) => format!( + "{class:?} is not permitted while the namespace fence is {} (operation {}, revision {})", + r.state, r.operation_id, r.revision + ), + StoredFence::Unavailable { reason, .. } => { + format!("the namespace's fence state cannot be established: {reason}") + } + StoredFence::None { .. } => format!("{class:?} is not permitted"), + } + } +} + +/// Where the marker of `namespace` lives, under the server's `dbs` directory. +pub fn marker_path(dbs_path: &Path, namespace: &NamespaceName) -> PathBuf { + dbs_path.join(namespace.as_str()).join(MARKER_FILE_NAME) +} + +/// Read a namespace's marker. `Ok(None)` when there is none; `Ok(Some(Err(_)))` when there is +/// one that cannot be decoded. +pub fn read_marker( + dbs_path: &Path, + namespace: &NamespaceName, +) -> io::Result>> { + match fs::read(marker_path(dbs_path, namespace)) { + Ok(bytes) => Ok(Some(FenceMarker::decode(&bytes))), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// Durably replace a namespace's marker with a copy of `record`: write a temporary file, fsync +/// it, rename it over the marker and fsync the directory. Creates the namespace directory if +/// it does not exist yet (a target that is being created). +pub fn write_marker(dbs_path: &Path, record: &NamespaceFenceRecord) -> io::Result<()> { + let path = marker_path(dbs_path, &record.namespace); + let dir = path.parent().expect("marker path has a parent"); + fs::create_dir_all(dir)?; + let tmp = dir.join(format!("{MARKER_FILE_NAME}.tmp")); + { + let mut file = File::create(&tmp)?; + file.write_all(&FenceMarker::for_record(record).encode())?; + file.sync_all()?; + } + fs::rename(&tmp, &path)?; + File::open(dir)?.sync_all()?; + Ok(()) +} + +/// Remove a namespace's marker, if it has one, and fsync its directory. +pub fn remove_marker(dbs_path: &Path, namespace: &NamespaceName) -> io::Result<()> { + let path = marker_path(dbs_path, namespace); + match fs::remove_file(&path) { + Ok(()) => File::open(path.parent().expect("marker path has a parent"))?.sync_all(), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// The namespace directories under `dbs_path` that hold a marker, in no particular order. A +/// directory whose name is not a valid namespace name is returned as its raw name, so the +/// caller can refuse to start rather than ignore it. +pub fn scan_markers(dbs_path: &Path) -> io::Result>> { + let entries = match fs::read_dir(dbs_path) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e), + }; + let mut out = Vec::new(); + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_dir() || !entry.path().join(MARKER_FILE_NAME).try_exists()? { + continue; + } + let raw = entry.file_name(); + out.push(match raw.to_str() { + Some(name) => { + NamespaceName::from_string(name.to_string()).map_err(|_| name.to_string()) + } + None => Err(raw.to_string_lossy().into_owned()), + }); + } + Ok(out) +} + +/// Whether the marker agrees with what the metastore says. Returned by [`read_fence`] so a +/// loader can repair a marker that fell behind (a crash between commit and marker write). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MarkerStatus { + /// The marker holds the stored record, or there is neither. + Current, + /// The metastore has a record and the marker is missing or older: rewrite it. + Stale, + /// The marker is what makes the namespace unavailable, or cannot be read. + Conflicting, +} + +struct RawFenceRow { + format_version: i64, + revision: i64, + record: Vec, +} + +fn read_raw_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + conn.query_row( + "SELECT format_version, revision, record FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + |row| { + Ok(RawFenceRow { + format_version: row.get(0)?, + revision: row.get(1)?, + record: row.get(2)?, + }) + }, + ) + .optional() +} + +/// The stored revision column of `namespace`'s fence row, whatever its payload says. +pub fn stored_revision( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + conn.query_row( + "SELECT revision FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get(0), + ) + .optional() +} + +/// Like [`stored_revision`], for a namespace name that is not a valid [`NamespaceName`]. +pub fn stored_revision_raw( + conn: &rusqlite::Connection, + namespace: &str, +) -> rusqlite::Result> { + conn.query_row( + "SELECT revision FROM namespace_fences WHERE namespace = ?1", + [namespace], + |row| row.get(0), + ) + .optional() +} + +fn decode_row(row: &RawFenceRow) -> Result { + let format_version = u32::try_from(row.format_version) + .map_err(|_| FenceDecodeError::UnsupportedFormatVersion(u32::MAX))?; + let revision = u64::try_from(row.revision) + .map_err(|_| FenceDecodeError::Invalid("negative revision column"))?; + NamespaceFenceRecord::decode(format_version, revision, &row.record) +} + +fn decode_detail(e: &FenceDecodeError) -> FenceDetail { + match e { + FenceDecodeError::UnsupportedFormatVersion(_) => FenceDetail::UnsupportedFormatVersion, + FenceDecodeError::Undecodable(_) | FenceDecodeError::Invalid(_) => { + FenceDetail::CorruptRecord + } + } +} + +/// Read the config row of `namespace`, if it has one. +pub fn read_config_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> Result, FenceStoreError> { + let bytes: Option> = conn + .query_row( + "SELECT config FROM namespace_configs WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get(0), + ) + .optional()?; + match bytes { + None => Ok(None), + Some(bytes) => match metadata::DatabaseConfig::decode(&bytes[..]) { + Ok(c) => Ok(Some(DatabaseConfig::from(&c))), + Err(e) => Err(FenceError::new( + FenceOutcome::FenceStateUnavailable, + format!("the namespace's config row cannot be decoded: {e}"), + ) + .with_detail(FenceDetail::CorruptRecord) + .into()), + }, + } +} + +fn config_row_exists( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result { + conn.query_row( + "SELECT count(*) FROM namespace_configs WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) +} + +/// Establish the fence of `namespace` from its row and its marker (section 5.6). +/// +/// Only a metastore that has the fence tables can hold a record; `dbs_path` is where the +/// namespace directories, and so the markers, are. +pub fn read_fence( + conn: &rusqlite::Connection, + dbs_path: &Path, + namespace: &NamespaceName, +) -> Result<(StoredFence, MarkerStatus), FenceStoreError> { + let row = read_raw_row(conn, namespace)?; + let marker = read_marker(dbs_path, namespace)?; + + let marker_record = match &marker { + Some(Ok(m)) => Some(m.record.clone()), + _ => None, + }; + + let Some(row) = row else { + return Ok(match marker { + None => ( + StoredFence::None { + namespace_exists: config_row_exists(conn, namespace)?, + }, + MarkerStatus::Current, + ), + Some(Err(e)) => ( + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be decoded: {e}"), + marker: None, + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) => { + let incomplete_creation = m.record.state == FenceState::TargetQuarantined + && m.record.revision == 1 + && !config_row_exists(conn, namespace)?; + let (detail, reason) = if incomplete_creation { + ( + FenceDetail::IncompleteTargetCreation, + "a target creation was interrupted before its metastore commit".to_string(), + ) + } else { + ( + FenceDetail::MetastoreBehindMarker, + format!( + "the marker records revision {} but the metastore has no fence record", + m.record.revision + ), + ) + }; + ( + StoredFence::Unavailable { + detail, + reason, + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ) + } + }); + }; + + let record = match decode_row(&row) { + Ok(record) => record, + Err(e) => { + return Ok(( + StoredFence::Unavailable { + detail: decode_detail(&e), + reason: format!("the fence record cannot be read: {e}"), + marker: marker_record, + }, + MarkerStatus::Conflicting, + )) + } + }; + + if record.namespace != *namespace { + return Ok(( + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence record names namespace `{}`", record.namespace), + marker: marker_record, + }, + MarkerStatus::Conflicting, + )); + } + + Ok(match marker { + Some(Ok(m)) if m.record.revision > record.revision => ( + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the marker records revision {} but the metastore has revision {}", + m.record.revision, record.revision + ), + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) if m.record.revision == record.revision && m.record != record => ( + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the marker and the metastore disagree at revision {}", + record.revision + ), + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) if m.record == record => (StoredFence::Record(record), MarkerStatus::Current), + // Missing, older, or unreadable while the metastore has a well-formed record: the + // metastore is authoritative and the marker is rewritten. + _ => (StoredFence::Record(record), MarkerStatus::Stale), + }) +} + +/// Look up the receipt for `(namespace, operation_id, command_id)`. +pub fn read_receipt( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, +) -> Result>, rusqlite::Error> { + let row: Option<(i64, Vec)> = conn + .query_row( + "SELECT format_version, receipt FROM namespace_fence_receipts + WHERE namespace = ?1 AND operation_id = ?2 AND command_id = ?3", + params![ + namespace.as_str(), + operation_id.to_string(), + command_id.to_string() + ], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + Ok(row.map(|(v, bytes)| decode_receipt(v, &bytes, namespace, operation_id, command_id))) +} + +fn decode_receipt( + format_version: i64, + bytes: &[u8], + namespace: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, +) -> Result { + let format_version = u32::try_from(format_version) + .map_err(|_| FenceDecodeError::UnsupportedFormatVersion(u32::MAX))?; + let receipt = CommandReceipt::decode(format_version, bytes)?; + if receipt.namespace != *namespace + || receipt.operation_id != operation_id + || receipt.command_id != command_id + { + return Err(FenceDecodeError::Invalid("receipt disagrees with its key")); + } + Ok(receipt) +} + +/// One stored receipt, as read for inspection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredReceipt { + pub operation_id: String, + pub command_id: String, + pub revision_after: i64, + pub applied_at_ms: i64, + pub receipt: Result, +} + +/// Every receipt of `namespace`, oldest first. +pub fn read_receipts( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT operation_id, command_id, format_version, revision_after, applied_at, receipt + FROM namespace_fence_receipts WHERE namespace = ?1 + ORDER BY applied_at, revision_after, operation_id, command_id", + )?; + let rows = stmt.query_map([namespace.as_str()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Vec>(5)?, + )) + })?; + let mut out = Vec::new(); + for row in rows { + let (op, cmd, version, revision_after, applied_at, bytes) = row?; + let receipt = match (Uuid::parse_str(&op), Uuid::parse_str(&cmd)) { + (Ok(op_id), Ok(cmd_id)) => decode_receipt(version, &bytes, namespace, op_id, cmd_id), + _ => Err(FenceDecodeError::Invalid("receipt key is not a uuid")), + }; + out.push(StoredReceipt { + operation_id: op, + command_id: cmd, + revision_after, + applied_at_ms: applied_at, + receipt, + }); + } + Ok(out) +} + +/// Compare-and-swap the fence row of `record.namespace`: it must currently have revision +/// `previous` (`None`: no row). The caller holds the write lock, so this only fails if the +/// caller read something other than what is stored, which is a bug; it is still checked. +pub fn write_record( + conn: &rusqlite::Connection, + record: &NamespaceFenceRecord, + previous: Option, +) -> Result<(), FenceStoreError> { + let revision = i64::try_from(record.revision).expect("revision fits in i64"); + let bytes = record.encode(); + let changed = match previous { + None => conn.execute( + "INSERT INTO namespace_fences (namespace, format_version, revision, record) + VALUES (?1, ?2, ?3, ?4)", + params![ + record.namespace.as_str(), + FENCE_FORMAT_VERSION, + revision, + bytes + ], + )?, + Some(previous) => conn.execute( + "UPDATE namespace_fences SET format_version = ?2, revision = ?3, record = ?4 + WHERE namespace = ?1 AND revision = ?5", + params![ + record.namespace.as_str(), + FENCE_FORMAT_VERSION, + revision, + bytes, + previous + ], + )?, + }; + if changed != 1 { + return Err(FenceError::new( + FenceOutcome::FenceRevisionMismatch, + "the stored fence record changed under the transition", + ) + .into()); + } + Ok(()) +} + +/// Store `receipt`, replacing any receipt with the same key (a finished drain replaces its +/// `DRAINING` receipt). +pub fn write_receipt( + conn: &rusqlite::Connection, + receipt: &CommandReceipt, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT OR REPLACE INTO namespace_fence_receipts + (namespace, operation_id, command_id, format_version, revision_after, applied_at, receipt) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + receipt.namespace.as_str(), + receipt.operation_id.to_string(), + receipt.command_id.to_string(), + FENCE_FORMAT_VERSION, + i64::try_from(receipt.revision_after).expect("revision fits in i64"), + receipt.applied_at_ms, + receipt.encode(), + ], + )?; + Ok(()) +} + +/// Prune receipts of operations other than `owner` that are older than `retention` at +/// `now_ms` (section 5.5). The owner's receipts are never pruned. +pub fn prune_receipts( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + owner: Uuid, + now_ms: i64, + retention: Duration, +) -> rusqlite::Result { + let cutoff = now_ms.saturating_sub(i64::try_from(retention.as_millis()).unwrap_or(i64::MAX)); + conn.execute( + "DELETE FROM namespace_fence_receipts + WHERE namespace = ?1 AND operation_id != ?2 AND applied_at < ?3", + params![namespace.as_str(), owner.to_string(), cutoff], + ) +} + +/// Remove every trace of `namespace`'s fence, for a delete of a namespace whose fence permits +/// it. Returns the number of receipts removed. +pub fn delete_fence( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result { + conn.execute( + "DELETE FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + )?; + conn.execute( + "DELETE FROM namespace_fence_receipts WHERE namespace = ?1", + [namespace.as_str()], + ) +} + +/// The config as it is stored while `blocks` is the legacy mirror: `config` with its +/// `block_*` fields replaced (section 13.2). +pub fn with_legacy_blocks(config: &DatabaseConfig, blocks: &LegacyBlocks) -> DatabaseConfig { + DatabaseConfig { + block_reads: blocks.block_reads, + block_writes: blocks.block_writes, + block_reason: blocks.block_reason.clone(), + ..config.clone() + } +} + +/// The `block_*` fields of `config`. +pub fn legacy_blocks_of(config: &DatabaseConfig) -> LegacyBlocks { + LegacyBlocks { + block_reads: config.block_reads, + block_writes: config.block_writes, + block_reason: config.block_reason.clone(), + } +} + +/// Upsert the config row of `namespace`. +pub fn write_config_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + config: &DatabaseConfig, +) -> rusqlite::Result<()> { + let encoded = metadata::DatabaseConfig::from(config).encode_to_vec(); + conn.execute( + "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) + ON CONFLICT(namespace) DO UPDATE SET config = excluded.config", + params![namespace.as_str(), encoded], + )?; + Ok(()) +} + +/// The namespace config a target is created with, before the legacy mirror is applied. +pub fn target_database_config(config: &TargetConfig) -> Result { + let invalid = |message: String| { + FenceError::new(FenceOutcome::FencePreconditionFailed, message) + .with_detail(FenceDetail::InvalidArgument) + }; + let mut out = DatabaseConfig::default(); + if let Some(bytes) = config.max_db_size { + out.max_db_pages = bytes / LIBSQL_PAGE_SIZE; + } + out.jwt_key = config.jwt_key.clone(); + if let Some(s) = config.txn_timeout_s { + out.txn_timeout = Some(Duration::from_secs(s)); + } + out.allow_attach = config.allow_attach; + if let Some(mode) = &config.durability_mode { + out.durability_mode = mode + .parse::() + .map_err(|()| invalid(format!("unknown durability mode `{mode}`")))?; + } + out.bottomless_db_id = config.bottomless_db_id.clone(); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::namespace::fence::record::tests as samples; + + fn conn() -> rusqlite::Connection { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute("PRAGMA foreign_keys=ON", ()).unwrap(); + conn.execute( + "CREATE TABLE namespace_configs (namespace TEXT NOT NULL PRIMARY KEY, config BLOB NOT NULL)", + (), + ) + .unwrap(); + create_tables(&conn).unwrap(); + conn + } + + fn sample() -> NamespaceFenceRecord { + samples::sample_record() + } + + #[test] + fn tables_are_additive_and_detectable() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + assert!(!tables_exist(&conn).unwrap()); + create_tables(&conn).unwrap(); + assert!(tables_exist(&conn).unwrap()); + // Idempotent. + create_tables(&conn).unwrap(); + } + + #[test] + fn record_round_trips_and_cas_checks_revision() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + // A second insert, or an update from the wrong revision, is refused. + assert!(write_record(&conn, &record, None).is_err()); + let mut next = record.clone(); + next.revision += 1; + let err = write_record(&conn, &next, Some(1)).unwrap_err(); + assert!( + matches!(err, FenceStoreError::Fence(e) if e.outcome() == FenceOutcome::FenceRevisionMismatch) + ); + write_record(&conn, &next, Some(2)).unwrap(); + + let (stored, marker) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(next)); + assert_eq!(marker, MarkerStatus::Stale); + } + + #[test] + fn fence_row_needs_a_config_row() { + let conn = conn(); + let err = write_record(&conn, &sample(), None).unwrap_err(); + assert!(matches!(err, FenceStoreError::Sqlite(_)), "{err:?}"); + } + + #[test] + fn delete_of_config_row_is_restricted_by_the_fence_row() { + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + assert!(conn + .execute( + "DELETE FROM namespace_configs WHERE namespace = ?1", + [record.namespace.as_str()] + ) + .is_err()); + delete_fence(&conn, &record.namespace).unwrap(); + conn.execute( + "DELETE FROM namespace_configs WHERE namespace = ?1", + [record.namespace.as_str()], + ) + .unwrap(); + } + + #[test] + fn marker_round_trips_and_is_compared() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + write_marker(dir.path(), &record).unwrap(); + assert!(!dir + .path() + .join(record.namespace.as_str()) + .join(".fence.tmp") + .exists()); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(record.clone())); + assert_eq!(status, MarkerStatus::Current); + + // A marker ahead of the metastore: the metastore went backwards. + let mut ahead = record.clone(); + ahead.revision += 1; + write_marker(dir.path(), &ahead).unwrap(); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { detail: FenceDetail::MetastoreBehindMarker, marker: Some(ref m), .. } if *m == ahead + )); + assert_eq!(status, MarkerStatus::Conflicting); + + // Same revision, different contents. + let mut forked = record.clone(); + forked.operation_id = Uuid::from_u128(77); + write_marker(dir.path(), &forked).unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored.state(), FenceState::UnknownUnavailable); + + // An undecodable marker beside a good record: the record wins and the marker is stale. + fs::write(marker_path(dir.path(), &record.namespace), b"garbage").unwrap(); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(record.clone())); + assert_eq!(status, MarkerStatus::Stale); + + // Marker without a row. + delete_fence(&conn, &record.namespace).unwrap(); + write_marker(dir.path(), &record).unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + .. + } + )); + fs::write(marker_path(dir.path(), &record.namespace), b"garbage").unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + marker: None, + .. + } + )); + } + + #[test] + fn corrupt_rows_are_unavailable() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + let ns = record.namespace.clone(); + write_config_row(&conn, &ns, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + + let set = |sql: &str| { + conn.execute(sql, [ns.as_str()]).unwrap(); + }; + let detail = || match read_fence(&conn, dir.path(), &ns).unwrap().0 { + StoredFence::Unavailable { detail, .. } => detail, + other => panic!("expected unavailable, got {other:?}"), + }; + + set("UPDATE namespace_fences SET format_version = 2 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::UnsupportedFormatVersion); + set("UPDATE namespace_fences SET format_version = 1, revision = 3 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + set("UPDATE namespace_fences SET revision = 2, record = x'ffff' WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + set("UPDATE namespace_fences SET revision = -1 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + + let stored = read_fence(&conn, dir.path(), &ns).unwrap().0; + for class in OperationClass::ALL { + let r = stored.permits(class); + match class { + OperationClass::Maintenance | OperationClass::Observability => assert!(r.is_ok()), + _ => { + let e = r.unwrap_err(); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + assert_eq!(e.detail(), Some(FenceDetail::CorruptRecord)); + } + } + } + } + + #[test] + fn receipts_round_trip_and_prune() { + let conn = conn(); + let record = sample(); + let ns = record.namespace.clone(); + let base = samples::sample_receipt(); + let owner = base.operation_id; + let other = Uuid::from_u128(0xbeef); + + let mut r_owner_old = base.clone(); + r_owner_old.applied_at_ms = 10; + let mut r_other_old = base.clone(); + r_other_old.operation_id = other; + r_other_old.applied_at_ms = 10; + let mut r_other_new = base.clone(); + r_other_new.operation_id = other; + r_other_new.command_id = Uuid::from_u128(0xc0de); + r_other_new.applied_at_ms = 5_000; + for r in [&r_owner_old, &r_other_old, &r_other_new] { + write_receipt(&conn, r).unwrap(); + } + assert_eq!( + read_receipt(&conn, &ns, owner, base.command_id) + .unwrap() + .unwrap() + .unwrap(), + r_owner_old + ); + assert!(read_receipt(&conn, &ns, owner, Uuid::from_u128(1234)) + .unwrap() + .is_none()); + + // Retention 1s at t=6s: only the other operation's old receipt goes. + let pruned = prune_receipts(&conn, &ns, owner, 6_000, Duration::from_secs(1)).unwrap(); + assert_eq!(pruned, 1); + let left: Vec<_> = read_receipts(&conn, &ns) + .unwrap() + .into_iter() + .map(|r| r.receipt.unwrap()) + .collect(); + assert_eq!(left, vec![r_owner_old.clone(), r_other_new]); + + // A receipt stored under the wrong key reads as corrupt. + conn.execute( + "UPDATE namespace_fence_receipts SET command_id = ?1 WHERE operation_id = ?2", + params![Uuid::from_u128(4321).to_string(), owner.to_string()], + ) + .unwrap(); + assert!(read_receipt(&conn, &ns, owner, Uuid::from_u128(4321)) + .unwrap() + .unwrap() + .is_err()); + } + + #[test] + fn target_config_conversion() { + let c = target_database_config(&TargetConfig { + max_db_size: Some(4096 * 10), + jwt_key: Some("k".into()), + txn_timeout_s: Some(7), + allow_attach: true, + durability_mode: Some("strong".into()), + bottomless_db_id: Some("b".into()), + }) + .unwrap(); + assert_eq!(c.max_db_pages, 10); + assert_eq!(c.jwt_key.as_deref(), Some("k")); + assert_eq!(c.txn_timeout, Some(Duration::from_secs(7))); + assert!(c.allow_attach); + assert_eq!(c.durability_mode, DurabilityMode::Strong); + assert_eq!(c.bottomless_db_id.as_deref(), Some("b")); + assert!(!c.block_reads && !c.block_writes); + + let e = target_database_config(&TargetConfig { + durability_mode: Some("nope".into()), + ..Default::default() + }) + .unwrap_err(); + assert_eq!(e.detail(), Some(FenceDetail::InvalidArgument)); + } +} diff --git a/libsql-server/src/namespace/fence/transition.rs b/libsql-server/src/namespace/fence/transition.rs new file mode 100644 index 0000000000..f1de43f747 --- /dev/null +++ b/libsql-server/src/namespace/fence/transition.rs @@ -0,0 +1,1794 @@ +//! The pure fence transition function (`docs/NAMESPACE_FENCE.md` sections 3.2 and 5.3). +//! +//! [`apply`] decides what a command does to a namespace's fence, given everything the store +//! read inside its transaction. It performs no I/O, takes no locks and reads no clock: the +//! store supplies the current record, the stored receipt for the request's +//! `(operation_id, command_id)` if there is one, and the facts in [`ApplyEnv`]. The store +//! persists whatever [`Decision::Apply`] returns, in one transaction, before anything is +//! published or answered. +//! +//! Checks run in this order, and the order is part of the contract: +//! +//! 1. **Replay.** A stored receipt with the same fingerprint is answered from the receipt +//! (`Replay`, or `Resume` for a drain still in progress), whatever has happened to the +//! record since. A stored receipt with a different fingerprint is `FENCE_COMMAND_CONFLICT`. +//! 2. **Unavailable state.** A record the server cannot establish refuses everything with +//! `FENCE_STATE_UNAVAILABLE`, except the two commands that can reconcile it: a replay of the +//! `CreateTargetQuarantined` that left the marker, and an adoption after a metastore +//! rollback. +//! 3. **Owner.** An unfinished record owned by another operation is +//! `FENCE_OWNED_BY_ANOTHER_OPERATION`. +//! 4. **Already applied.** A command from the owner whose goal state the record is already in +//! is `ALREADY_APPLIED`: a receipt is stored, the record and its revision do not change. +//! This is checked before the revision, because the caller's stated expectation is +//! typically the state before a response it never received. +//! 5. **Transition.** Role, then legality of the transition from the current state +//! (`INVALID_FENCE_TRANSITION`). +//! 6. **Expectation.** `expected_state` and `expected_revision` (`FENCE_REVISION_MISMATCH`). +//! 7. **Preconditions** of the command (`FENCE_PRECONDITION_FAILED`). +//! +//! A drain that starts in `apply` (`DRAINING`) is finished by [`complete_drain`], once the +//! controller has proven that the work admitted earlier has ended. + +use uuid::Uuid; + +use super::command::{ + AdoptArgs, CommandKind, FenceCommand, FenceRequest, ValidationResult, + MAX_VALIDATION_SUMMARY_BYTES, +}; +use super::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::record::{ + Adoption, CommandReceipt, FrozenBoundary, LegacyBlocks, NamespaceFenceRecord, + NamespaceIdentity, ServerIdentity, ValidationRecord, ValidationSnapshot, +}; +use super::state::{FenceState, Role}; + +/// What the store knows about a namespace's fence. +#[derive(Debug, Clone, Copy)] +pub enum CurrentFence<'a> { + /// No record. `namespace_exists` distinguishes `UNFENCED` from `ABSENT`. + None { + namespace_exists: bool, + }, + Record(&'a NamespaceFenceRecord), + /// The control state cannot be established. `marker` is the record the namespace's marker + /// file holds, when it has a readable one. + Unavailable { + detail: FenceDetail, + marker: Option<&'a NamespaceFenceRecord>, + }, +} + +impl CurrentFence<'_> { + pub fn state(&self) -> FenceState { + match self { + CurrentFence::None { + namespace_exists: true, + } => FenceState::Unfenced, + CurrentFence::None { + namespace_exists: false, + } => FenceState::Absent, + CurrentFence::Record(r) => r.state, + CurrentFence::Unavailable { .. } => FenceState::UnknownUnavailable, + } + } + + pub fn revision(&self) -> u64 { + match self { + CurrentFence::None { .. } => 0, + CurrentFence::Record(r) => r.revision, + CurrentFence::Unavailable { marker, .. } => marker.map_or(0, |m| m.revision), + } + } +} + +/// Facts `apply` needs that are not in the record. The store fills them from inside the same +/// transaction and the live namespace. +#[derive(Debug, Clone)] +pub struct ApplyEnv { + /// Wall-clock time, in milliseconds since the Unix epoch. Informational only: nothing in + /// the fence expires. + pub now_ms: i64, + pub server: ServerIdentity, + /// The namespace's current replication log id, if it exists and has one. + pub namespace_log_id: Option, + /// Whether the namespace is a shared schema or linked to one. + pub shared_schema: bool, + /// The namespace config's current `block_*` values, saved when a source is acquired and + /// restored when it is released. + pub legacy_blocks: LegacyBlocks, + /// A fresh id, used as `target_incarnation_id` by `CreateTargetQuarantined`. + pub new_incarnation_id: Uuid, + /// Whether the request carried the configured adoption key. + pub adoption_authorised: bool, + /// What the server observed of the target, for `RecordTargetValidation`. + pub validation_snapshot: Option, +} + +/// What a command does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decision { + /// The command was applied before and its answer is final: return this receipt, with + /// `replayed: true`. Nothing is written. + Replay(CommandReceipt), + /// The same command started a drain that has not been completed: resume it. Nothing is + /// written. + Resume(CommandReceipt), + /// Persist `record` (when `Some`; `None` leaves the record as it is) and `receipt` in one + /// transaction, then answer `receipt.outcome`. A `DRAINING` receipt means the controller + /// must now run the drain and finish it with [`complete_drain`]. + Apply { + record: Option, + receipt: CommandReceipt, + }, +} + +/// Evidence, gathered by the controller, that a drain is complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DrainCompletion { + /// No normal writer holds or can take the write slot; `boundary` was read with the slot + /// free, after the last commit published its frame. + SourceWrites { boundary: FrozenBoundary }, + /// Every SQL, dump and replication read lease has been released. + SourceReads, + /// Every import writer has finished and no capability writer holds the slot. + TargetImport, +} + +fn err(outcome: FenceOutcome, message: impl Into) -> FenceError { + FenceError::new(outcome, message) +} + +fn invalid(message: impl Into) -> FenceError { + err(FenceOutcome::InvalidFenceTransition, message) +} + +fn precondition(detail: FenceDetail, message: impl Into) -> FenceError { + err(FenceOutcome::FencePreconditionFailed, message).with_detail(detail) +} + +/// The role a command acts on. `None` for adoption, which acts on either. +fn command_role(kind: CommandKind) -> Option { + match kind { + CommandKind::AcquireSourceWriteFence + | CommandKind::SetSourceReadFence + | CommandKind::ClearSourceReadFence + | CommandKind::ReleaseSourceWriteFence => Some(Role::Source), + CommandKind::CreateTargetQuarantined + | CommandKind::SealTargetImport + | CommandKind::RecordTargetValidation + | CommandKind::PublishTargetReadableWriteFenced + | CommandKind::EnableTargetWrites + | CommandKind::AbortQuarantinedTarget => Some(Role::Target), + CommandKind::AdoptFence => None, + } +} + +/// The state a legal command moves a record from `from` to, and the receipt outcome. This is +/// the transition graph of section 3.2, minus the drain completions and adoption. +fn transition(kind: CommandKind, from: FenceState) -> Option<(FenceState, FenceOutcome)> { + use CommandKind as K; + use FenceOutcome::{Applied, Draining}; + use FenceState as S; + + Some(match (kind, from) { + (K::AcquireSourceWriteFence, S::Unfenced | S::Released | S::TargetWritable) => { + (S::SourceDraining, Draining) + } + (K::SetSourceReadFence, S::SourceWriteFenced) => (S::SourceReadDraining, Draining), + (K::ClearSourceReadFence, S::SourceReadDraining | S::SourceReadFenced) => { + (S::SourceWriteFenced, Applied) + } + (K::ReleaseSourceWriteFence, S::SourceDraining | S::SourceWriteFenced) => { + (S::Released, Applied) + } + (K::CreateTargetQuarantined, S::Absent) => (S::TargetQuarantined, Applied), + (K::SealTargetImport, S::TargetQuarantined) => (S::TargetImportDraining, Draining), + (K::RecordTargetValidation, S::TargetValidating) => (S::TargetValidating, Applied), + (K::PublishTargetReadableWriteFenced, S::TargetValidating) => { + (S::TargetWriteFenced, Applied) + } + (K::EnableTargetWrites, S::TargetWriteFenced) => (S::TargetWritable, Applied), + ( + K::AbortQuarantinedTarget, + S::TargetQuarantined + | S::TargetImportDraining + | S::TargetValidating + | S::TargetWriteFenced, + ) => (S::TargetAborted, Applied), + _ => return None, + }) +} + +/// The draining state a command's drain runs in, for commands that drain. +fn drain_state(kind: CommandKind) -> Option { + match kind { + CommandKind::AcquireSourceWriteFence => Some(FenceState::SourceDraining), + CommandKind::SetSourceReadFence => Some(FenceState::SourceReadDraining), + CommandKind::SealTargetImport => Some(FenceState::TargetImportDraining), + _ => None, + } +} + +fn receipt( + request: &FenceRequest, + env: &ApplyEnv, + outcome: FenceOutcome, + revision_before: u64, + revision_after: u64, + state_after: FenceState, +) -> CommandReceipt { + CommandReceipt { + namespace: request.namespace.clone(), + operation_id: request.operation_id, + command_id: request.command_id, + command: request.command.kind(), + fingerprint: request.fingerprint(), + outcome, + revision_before, + revision_after, + state_after, + applied_at_ms: env.now_ms, + instance_id: env.server.instance_id, + adoption: None, + } +} + +/// Decide what `request` does to the namespace's fence. See the module documentation for the +/// order of the checks. +/// +/// `existing` is the stored receipt for `(request.namespace, request.operation_id, +/// request.command_id)`, if any. +pub fn apply( + current: CurrentFence<'_>, + existing: Option<&CommandReceipt>, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let kind = request.command.kind(); + + // 1. Replay, before anything about the record is looked at. + if let Some(existing) = existing { + debug_assert_eq!(existing.operation_id, request.operation_id); + debug_assert_eq!(existing.command_id, request.command_id); + if existing.fingerprint != request.fingerprint() { + return Err(err( + FenceOutcome::FenceCommandConflict, + format!( + "command {} was already used for a different request", + request.command_id + ), + )); + } + return Ok(if existing.is_final() { + Decision::Replay(existing.clone()) + } else { + Decision::Resume(existing.clone()) + }); + } + + // 2. A state the server cannot establish. + let record = match current { + CurrentFence::None { .. } => None, + CurrentFence::Record(record) => Some(record), + CurrentFence::Unavailable { detail, marker } => { + return apply_unavailable(detail, marker, request, env); + } + }; + + if let FenceCommand::AdoptFence(args) = &request.command { + return apply_adopt(current, record, args, request, env); + } + + if let Some(record) = record { + // 3. Owner. + let finished = record.state.is_operation_finished(); + if !finished && record.operation_id != request.operation_id { + return Err(err( + FenceOutcome::FenceOwnedByAnotherOperation, + format!( + "namespace fence is owned by operation {}", + record.operation_id + ), + )); + } + + let own = record.operation_id == request.operation_id; + + // 4. Already applied. + if own && kind.goal_state() == Some(record.state) { + let receipt = receipt( + request, + env, + FenceOutcome::AlreadyApplied, + record.revision, + record.revision, + record.state, + ); + return Ok(Decision::Apply { + record: None, + receipt, + }); + } + + // The owner joining its own drain under a new command id (for example after + // adoption, or after losing the original command id): nothing changes but the receipt, + // and the controller resumes the drain. + if own && drain_state(kind) == Some(record.state) { + check_expectation(current, request)?; + let receipt = receipt( + request, + env, + FenceOutcome::Draining, + record.revision, + record.revision, + record.state, + ); + return Ok(Decision::Apply { + record: None, + receipt, + }); + } + + if own && finished { + return Err(invalid(format!( + "operation {} has finished with this namespace ({})", + record.operation_id, record.state + )) + .with_detail(FenceDetail::OperationFinished)); + } + } + + // `CreateTargetQuarantined` needs a name nobody uses, whatever the record says. + if kind == CommandKind::CreateTargetQuarantined && current.state() != FenceState::Absent { + return Err(precondition( + FenceDetail::NamespaceExists, + "the namespace already exists", + )); + } + + // 5. Role and transition. + let from = current.state(); + let role = command_role(kind).expect("adoption is handled above"); + let current_role = match from { + // A published target, or a released source, may be acquired as a source by a new + // operation. + FenceState::Released | FenceState::TargetWritable | FenceState::Unfenced => { + Some(Role::Source) + } + FenceState::Absent => Some(Role::Target), + other => other.role(), + }; + if current_role != Some(role) { + return Err( + invalid(format!("{kind} does not apply to a namespace in {from}")) + .with_detail(FenceDetail::RoleMismatch), + ); + } + let Some((to, outcome)) = transition(kind, from) else { + return Err(invalid(format!("{kind} is not a transition from {from}"))); + }; + + // 6. Expectation. + check_expectation(current, request)?; + + // 7. Preconditions, and the next record. + let revision_before = current.revision(); + let revision_after = revision_before + 1; + let mut next = match record { + Some(record) if !record.state.is_operation_finished() => record.clone(), + _ => fresh_record(request, env, role), + }; + + match &request.command { + FenceCommand::AcquireSourceWriteFence { + expected_log_id, + drain_policy, + } => { + if env.shared_schema { + return Err(precondition( + FenceDetail::SharedSchemaUnsupported, + "shared-schema namespaces cannot be fenced", + )); + } + if env.namespace_log_id != Some(*expected_log_id) { + return Err(precondition( + FenceDetail::NamespaceIdentityMismatch, + "the namespace's replication log id is not the one the caller observed", + )); + } + next.identity = NamespaceIdentity { + log_id: Some(*expected_log_id), + target_incarnation_id: None, + }; + next.legacy_blocks = env.legacy_blocks.clone(); + next.drain_policy = *drain_policy; + next.drain_started_at_ms = Some(env.now_ms); + } + FenceCommand::SetSourceReadFence { drain_policy } + | FenceCommand::SealTargetImport { drain_policy } => { + next.drain_policy = *drain_policy; + next.drain_started_at_ms = Some(env.now_ms); + } + FenceCommand::ClearSourceReadFence + | FenceCommand::ReleaseSourceWriteFence + | FenceCommand::EnableTargetWrites + | FenceCommand::AbortQuarantinedTarget => { + next.drain_policy = None; + next.drain_started_at_ms = None; + } + FenceCommand::CreateTargetQuarantined { .. } => { + next.identity = NamespaceIdentity { + log_id: None, + target_incarnation_id: Some(env.new_incarnation_id), + }; + next.legacy_blocks = LegacyBlocks::default(); + } + FenceCommand::RecordTargetValidation { result, summary } => { + if summary.len() > MAX_VALIDATION_SUMMARY_BYTES { + return Err(precondition( + FenceDetail::InvalidArgument, + format!( + "validation summary is longer than {MAX_VALIDATION_SUMMARY_BYTES} bytes" + ), + )); + } + next.validation = Some(ValidationRecord { + operation_id: request.operation_id, + command_id: request.command_id, + result: *result, + summary: summary.clone(), + snapshot: env.validation_snapshot, + recorded_at_ms: env.now_ms, + }); + } + FenceCommand::PublishTargetReadableWriteFenced => { + let validated = next.validation.as_ref().is_some_and(|v| { + v.result == ValidationResult::Ok && v.operation_id == next.operation_id + }); + if !validated { + return Err(precondition( + FenceDetail::ValidationReceiptRequired, + "publication requires a successful validation receipt from the owning operation", + )); + } + } + FenceCommand::AdoptFence(_) => unreachable!("handled above"), + } + + next.state = to; + next.revision = revision_after; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = request.command_id; + next.written_by = env.server.clone(); + + let receipt = receipt(request, env, outcome, revision_before, revision_after, to); + Ok(Decision::Apply { + record: Some(next), + receipt, + }) +} + +/// A record for an operation that is starting on this namespace. +fn fresh_record(request: &FenceRequest, env: &ApplyEnv, role: Role) -> NamespaceFenceRecord { + NamespaceFenceRecord { + namespace: request.namespace.clone(), + role, + state: FenceState::Unfenced, + revision: 0, + operation_id: request.operation_id, + identity: NamespaceIdentity::default(), + drain_policy: None, + drain_started_at_ms: None, + frozen_boundary: None, + validation: None, + legacy_blocks: LegacyBlocks::default(), + created_at_ms: env.now_ms, + last_transition_at_ms: env.now_ms, + last_command_id: request.command_id, + written_by: env.server.clone(), + adoptions: Vec::new(), + } +} + +fn check_expectation(current: CurrentFence<'_>, request: &FenceRequest) -> Result<(), FenceError> { + let (state, revision) = (current.state(), current.revision()); + if request.expected_state != state || request.expected_revision != revision { + return Err(err( + FenceOutcome::FenceRevisionMismatch, + format!( + "expected {} at revision {}, found {} at revision {}", + request.expected_state, request.expected_revision, state, revision + ), + )); + } + Ok(()) +} + +fn apply_unavailable( + detail: FenceDetail, + marker: Option<&NamespaceFenceRecord>, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let unavailable = || { + err( + FenceOutcome::FenceStateUnavailable, + "the namespace's fence state cannot be established", + ) + .with_detail(detail) + }; + + match (&request.command, detail, marker) { + // A crash between writing the marker and committing the target's rows: only the same + // command completes the creation. + ( + FenceCommand::CreateTargetQuarantined { .. }, + FenceDetail::IncompleteTargetCreation, + Some(m), + ) if m.state == FenceState::TargetQuarantined + && m.revision == 1 + && m.operation_id == request.operation_id + && m.last_command_id == request.command_id => + { + let decision = apply( + CurrentFence::None { + namespace_exists: false, + }, + None, + request, + &ApplyEnv { + new_incarnation_id: m + .identity + .target_incarnation_id + .unwrap_or(env.new_incarnation_id), + ..env.clone() + }, + )?; + Ok(decision) + } + // The metastore went backwards: adoption re-establishes the record the marker last + // recorded (it is written only after a commit), under the adopting operation. + (FenceCommand::AdoptFence(args), FenceDetail::MetastoreBehindMarker, Some(m)) => { + apply_adopt(CurrentFence::Record(m), Some(m), args, request, env) + } + _ => Err(unavailable()), + } +} + +fn apply_adopt( + current: CurrentFence<'_>, + record: Option<&NamespaceFenceRecord>, + args: &AdoptArgs, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let Some(record) = record else { + return Err(invalid("there is no fence to adopt")); + }; + if record.state.is_operation_finished() { + return Err( + invalid(format!("a fence in {} cannot be adopted", record.state)) + .with_detail(FenceDetail::OperationFinished), + ); + } + if args.current_operation_id != record.operation_id { + return Err(err( + FenceOutcome::FenceOwnedByAnotherOperation, + format!( + "namespace fence is owned by operation {}", + record.operation_id + ), + )); + } + if request.operation_id == record.operation_id { + return Err(invalid("an operation cannot adopt its own fence")); + } + check_expectation(current, request)?; + if !env.adoption_authorised { + return Err(precondition( + FenceDetail::AdoptionNotAuthorised, + "adoption requires the configured adoption key", + )); + } + let approvers: Vec<&str> = args.approvers.iter().map(|a| a.trim()).collect(); + let two_distinct = approvers.len() == 2 + && approvers.iter().all(|a| !a.is_empty()) + && approvers[0] != approvers[1]; + if !two_distinct || args.incident_ref.trim().is_empty() || args.reason.trim().is_empty() { + return Err(precondition( + FenceDetail::AdoptionNotAuthorised, + "adoption requires two distinct approvers, an incident reference and a reason", + )); + } + + let revision_after = record.revision + 1; + let adoption = Adoption { + previous_operation_id: record.operation_id, + new_operation_id: request.operation_id, + command_id: request.command_id, + approvers: args.approvers.clone(), + incident_ref: args.incident_ref.clone(), + reason: args.reason.clone(), + at_ms: env.now_ms, + revision: revision_after, + }; + + // Ownership changes and nothing else: the state, and so every gate, stays as it was. + let mut next = record.clone(); + next.operation_id = request.operation_id; + next.revision = revision_after; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = request.command_id; + next.written_by = env.server.clone(); + next.adoptions.push(adoption.clone()); + + let mut receipt = receipt( + request, + env, + FenceOutcome::Applied, + record.revision, + revision_after, + record.state, + ); + receipt.adoption = Some(adoption); + Ok(Decision::Apply { + record: Some(next), + receipt, + }) +} + +/// Finish a drain that `apply` started. `receipt` is the owning operation's `DRAINING` receipt +/// for it; the returned receipt replaces it (same key) with the final `APPLIED` answer. +pub fn complete_drain( + record: &NamespaceFenceRecord, + receipt: &CommandReceipt, + completion: DrainCompletion, + env: &ApplyEnv, +) -> Result<(NamespaceFenceRecord, CommandReceipt), FenceError> { + if receipt.outcome != FenceOutcome::Draining || receipt.operation_id != record.operation_id { + return Err(invalid( + "there is no drain of the owning operation to complete", + )); + } + + let (from, to) = match completion { + DrainCompletion::SourceWrites { .. } => { + (FenceState::SourceDraining, FenceState::SourceWriteFenced) + } + DrainCompletion::SourceReads => { + (FenceState::SourceReadDraining, FenceState::SourceReadFenced) + } + DrainCompletion::TargetImport => ( + FenceState::TargetImportDraining, + FenceState::TargetValidating, + ), + }; + if record.state != from || drain_state(receipt.command) != Some(from) { + return Err(invalid(format!( + "{} cannot complete a drain from {}", + receipt.command, record.state + ))); + } + + let mut next = record.clone(); + if let DrainCompletion::SourceWrites { boundary } = completion { + if record.identity.log_id != Some(boundary.log_id) { + return Err(precondition( + FenceDetail::NamespaceIdentityMismatch, + "the frozen boundary belongs to a different replication log", + )); + } + next.frozen_boundary = Some(boundary); + } + next.state = to; + next.revision = record.revision + 1; + next.drain_policy = None; + next.drain_started_at_ms = None; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = receipt.command_id; + next.written_by = env.server.clone(); + + let mut final_receipt = receipt.clone(); + final_receipt.outcome = FenceOutcome::Applied; + final_receipt.revision_after = next.revision; + final_receipt.state_after = to; + final_receipt.applied_at_ms = env.now_ms; + final_receipt.instance_id = env.server.instance_id; + + Ok((next, final_receipt)) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::namespace::fence::command::{DrainPolicy, OnDeadline, TargetConfig}; + use crate::namespace::fence::record::{FenceMarker, FENCE_FORMAT_VERSION}; + use crate::namespace::NamespaceName; + + use FenceOutcome as O; + use FenceState as S; + + const LOG: Uuid = Uuid::from_u128(0x10); + const INCARNATION: Uuid = Uuid::from_u128(0x20); + const OP: Uuid = Uuid::from_u128(0xa); + const OTHER_OP: Uuid = Uuid::from_u128(0xb); + + fn env() -> ApplyEnv { + ApplyEnv { + now_ms: 1_000, + server: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(0x99), + }, + namespace_log_id: Some(LOG), + shared_schema: false, + legacy_blocks: LegacyBlocks::default(), + new_incarnation_id: INCARNATION, + adoption_authorised: false, + validation_snapshot: None, + } + } + + fn policy() -> Option { + Some(DrainPolicy { + deadline_ms: 1_000, + on_deadline: OnDeadline::Fail, + }) + } + + fn command(kind: CommandKind) -> FenceCommand { + match kind { + CommandKind::AcquireSourceWriteFence => FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: policy(), + }, + CommandKind::SetSourceReadFence => FenceCommand::SetSourceReadFence { + drain_policy: policy(), + }, + CommandKind::ClearSourceReadFence => FenceCommand::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence => FenceCommand::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined => FenceCommand::CreateTargetQuarantined { + config: TargetConfig::default(), + }, + CommandKind::SealTargetImport => FenceCommand::SealTargetImport { + drain_policy: policy(), + }, + CommandKind::RecordTargetValidation => FenceCommand::RecordTargetValidation { + result: ValidationResult::Ok, + summary: "row counts match".into(), + }, + CommandKind::PublishTargetReadableWriteFenced => { + FenceCommand::PublishTargetReadableWriteFenced + } + CommandKind::EnableTargetWrites => FenceCommand::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget => FenceCommand::AbortQuarantinedTarget, + CommandKind::AdoptFence => FenceCommand::AdoptFence(AdoptArgs { + current_operation_id: OP, + approvers: vec!["alice".into(), "bob".into()], + incident_ref: "INC-1".into(), + reason: "control plane lost".into(), + }), + } + } + + /// A little driver that plays the store: it keeps the record and receipts and applies + /// decisions the way the store will. + #[derive(Default)] + struct Harness { + namespace_exists: bool, + record: Option, + receipts: Vec, + next_command: u128, + } + + impl Harness { + fn source() -> Self { + Self { + namespace_exists: true, + ..Default::default() + } + } + + fn target() -> Self { + Self::default() + } + + fn current(&self) -> CurrentFence<'_> { + match &self.record { + Some(r) => CurrentFence::Record(r), + None => CurrentFence::None { + namespace_exists: self.namespace_exists, + }, + } + } + + fn request(&mut self, op: Uuid, command: FenceCommand) -> FenceRequest { + self.next_command += 1; + FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: op, + command_id: Uuid::from_u128(0x1000 + self.next_command), + expected_state: self.current().state(), + expected_revision: self.current().revision(), + command, + } + } + + fn lookup(&self, request: &FenceRequest) -> Option<&CommandReceipt> { + self.receipts.iter().find(|r| { + r.operation_id == request.operation_id && r.command_id == request.command_id + }) + } + + fn decide(&self, request: &FenceRequest, env: &ApplyEnv) -> Result { + apply(self.current(), self.lookup(request), request, env) + } + + fn persist(&mut self, decision: &Decision) { + if let Decision::Apply { record, receipt } = decision { + if let Some(record) = record { + // Everything apply writes must survive the durable encoding. + let decoded = NamespaceFenceRecord::decode( + FENCE_FORMAT_VERSION, + record.revision, + &record.encode(), + ) + .unwrap(); + assert_eq!(&decoded, record); + if let Some(old) = &self.record { + assert!(record.revision > old.revision, "revision must increase"); + } + self.record = Some(record.clone()); + } + self.store_receipt(receipt.clone()); + } + } + + fn store_receipt(&mut self, receipt: CommandReceipt) { + self.receipts.retain(|r| { + !(r.operation_id == receipt.operation_id && r.command_id == receipt.command_id) + }); + self.receipts.push(receipt); + } + + /// Send a fresh command with correct expectations and persist the result. + fn run(&mut self, op: Uuid, kind: CommandKind) -> Result { + let request = self.request(op, command(kind)); + self.run_request(&request, &env()) + } + + fn run_request( + &mut self, + request: &FenceRequest, + env: &ApplyEnv, + ) -> Result { + let decision = self.decide(request, env)?; + self.persist(&decision); + Ok(decision) + } + + fn complete(&mut self, completion: DrainCompletion) { + let record = self.record.clone().unwrap(); + let receipt = self + .receipts + .iter() + .find(|r| r.outcome == O::Draining && r.command_id == record.last_command_id) + .or_else(|| { + self.receipts + .iter() + .rev() + .find(|r| r.outcome == O::Draining) + }) + .unwrap() + .clone(); + let (next, receipt) = complete_drain(&record, &receipt, completion, &env()).unwrap(); + assert_eq!(next.revision, record.revision + 1); + self.record = Some(next); + self.store_receipt(receipt); + } + + fn state(&self) -> FenceState { + self.current().state() + } + + fn revision(&self) -> u64 { + self.current().revision() + } + + /// Drive the harness into `state` with operation `OP`. + fn in_state(state: FenceState) -> Self { + use CommandKind as K; + let boundary = DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 42, + }, + }; + let mut h = if state.role() == Some(Role::Target) || state == S::Absent { + Self::target() + } else { + Self::source() + }; + let steps: &[&dyn Fn(&mut Harness)] = match state { + S::Unfenced | S::Absent => &[], + S::SourceDraining => &[&|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap())], + S::SourceWriteFenced => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + ], + S::SourceReadDraining => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::SetSourceReadFence).unwrap()), + ], + S::SourceReadFenced => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::SetSourceReadFence).unwrap()), + &|h| h.complete(DrainCompletion::SourceReads), + ], + S::Released => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::ReleaseSourceWriteFence).unwrap()), + ], + S::TargetQuarantined => { + &[&|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap())] + } + S::TargetImportDraining => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + ], + S::TargetValidating => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + ], + S::TargetWriteFenced => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + &|h| drop(h.run(OP, K::RecordTargetValidation).unwrap()), + &|h| drop(h.run(OP, K::PublishTargetReadableWriteFenced).unwrap()), + ], + S::TargetWritable => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + &|h| drop(h.run(OP, K::RecordTargetValidation).unwrap()), + &|h| drop(h.run(OP, K::PublishTargetReadableWriteFenced).unwrap()), + &|h| drop(h.run(OP, K::EnableTargetWrites).unwrap()), + ], + S::TargetAborted => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::AbortQuarantinedTarget).unwrap()), + ], + S::UnknownUnavailable => panic!("not reachable by transitions"), + }; + for step in steps { + step(&mut h); + } + assert_eq!(h.state(), state); + h + } + } + + fn outcome_of(result: &Result) -> FenceOutcome { + match result { + Ok(Decision::Apply { receipt, .. }) => receipt.outcome, + Ok(Decision::Replay(r)) | Ok(Decision::Resume(r)) => r.outcome, + Err(e) => e.outcome(), + } + } + + fn state_after(result: &Result) -> Option { + match result { + Ok(Decision::Apply { receipt, .. }) => Some(receipt.state_after), + _ => None, + } + } + + const RECORD_STATES: [FenceState; 13] = [ + S::Unfenced, + S::Absent, + S::SourceDraining, + S::SourceWriteFenced, + S::SourceReadDraining, + S::SourceReadFenced, + S::Released, + S::TargetQuarantined, + S::TargetImportDraining, + S::TargetValidating, + S::TargetWriteFenced, + S::TargetWritable, + S::TargetAborted, + ]; + + /// Every (state, command) pair from the owning operation, with correct expectations, + /// against the full expected table: the legal transitions of section 3.2, the + /// `ALREADY_APPLIED` goal states, drain joins, and a refusal for everything else. + #[test] + fn exhaustive_owner_commands() { + use CommandKind as K; + + let expected = + |state: FenceState, kind: CommandKind| -> (FenceOutcome, Option) { + if kind == K::AdoptFence { + // Adopting one's own fence is never allowed; finished fences cannot be + // adopted; no record has nothing to adopt. + return (O::InvalidFenceTransition, None); + } + if kind == K::CreateTargetQuarantined { + return match state { + S::Absent => (O::Applied, Some(S::TargetQuarantined)), + S::TargetQuarantined => (O::AlreadyApplied, Some(S::TargetQuarantined)), + // The owner has finished with the namespace. + s if s.is_operation_finished() => (O::InvalidFenceTransition, None), + // The name is in use. + _ => (O::FencePreconditionFailed, None), + }; + } + if kind.goal_state() == Some(state) { + return (O::AlreadyApplied, Some(state)); + } + if drain_state(kind) == Some(state) { + return (O::Draining, Some(state)); + } + if state.is_operation_finished() && state != S::Unfenced { + return (O::InvalidFenceTransition, None); + } + match transition(kind, state) { + Some((to, outcome)) => { + if kind == K::PublishTargetReadableWriteFenced { + // No validation receipt has been recorded on this path. + (O::FencePreconditionFailed, None) + } else { + (outcome, Some(to)) + } + } + None => (O::InvalidFenceTransition, None), + } + }; + + for state in RECORD_STATES { + for kind in CommandKind::ALL { + // A target driven to TARGET_VALIDATING has no validation receipt yet. + let mut h = Harness::in_state(state); + let result = h.run(OP, kind); + let (outcome, to) = expected(state, kind); + assert_eq!(outcome_of(&result), outcome, "{state} {kind}: {result:?}"); + assert_eq!(state_after(&result), to, "{state} {kind}"); + } + } + } + + #[test] + fn source_happy_path() { + let mut h = Harness::source(); + let r = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap(); + let Decision::Apply { record, receipt } = r else { + panic!() + }; + let record = record.unwrap(); + assert_eq!(record.state, S::SourceDraining); + assert_eq!(record.revision, 1); + assert_eq!(receipt.outcome, O::Draining); + assert_eq!((receipt.revision_before, receipt.revision_after), (0, 1)); + assert_eq!(record.identity.log_id, Some(LOG)); + assert!(!record.write_admission().is_open()); + assert!(record.read_admission().is_open()); + + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 7, + }, + }); + assert_eq!(h.state(), S::SourceWriteFenced); + assert_eq!(h.revision(), 2); + assert_eq!( + h.record.as_ref().unwrap().frozen_boundary.unwrap().frame_no, + 7 + ); + let acquire_receipt = h + .receipts + .iter() + .find(|r| r.command == CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert_eq!(acquire_receipt.outcome, O::Applied); + assert_eq!(acquire_receipt.revision_after, 2); + + h.run(OP, CommandKind::SetSourceReadFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::SourceReadDraining, 3)); + h.complete(DrainCompletion::SourceReads); + assert_eq!((h.state(), h.revision()), (S::SourceReadFenced, 4)); + h.run(OP, CommandKind::ClearSourceReadFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::SourceWriteFenced, 5)); + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::Released, 6)); + assert!(h.record.as_ref().unwrap().write_admission().is_open()); + + // A new operation may acquire the released namespace; the revision keeps counting. + let r = h + .run(OTHER_OP, CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert!(matches!(r, Decision::Apply { .. })); + let record = h.record.as_ref().unwrap(); + assert_eq!( + (record.state, record.revision, record.operation_id), + (S::SourceDraining, 7, OTHER_OP) + ); + assert!(record.frozen_boundary.is_none()); + } + + #[test] + fn target_happy_path() { + let mut h = Harness::target(); + h.run(OP, CommandKind::CreateTargetQuarantined).unwrap(); + let record = h.record.clone().unwrap(); + assert_eq!( + (record.role, record.state, record.revision), + (Role::Target, S::TargetQuarantined, 1) + ); + assert_eq!(record.identity.target_incarnation_id, Some(INCARNATION)); + assert!(!record.read_admission().is_open()); + + h.run(OP, CommandKind::SealTargetImport).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetImportDraining, 2)); + h.complete(DrainCompletion::TargetImport); + assert_eq!((h.state(), h.revision()), (S::TargetValidating, 3)); + + // Publication needs a successful validation receipt first. + let err = h + .run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap_err(); + assert_eq!(err.outcome(), O::FencePreconditionFailed); + assert_eq!(err.detail(), Some(FenceDetail::ValidationReceiptRequired)); + + let failed = h.request( + OP, + FenceCommand::RecordTargetValidation { + result: ValidationResult::Failed, + summary: "mismatch".into(), + }, + ); + h.run_request(&failed, &env()).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetValidating, 4)); + let err = h + .run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::ValidationReceiptRequired)); + + let snapshot = ValidationSnapshot { + log_id: LOG, + frame_no: 9, + page_count: 3, + }; + let ok = h.request(OP, command(CommandKind::RecordTargetValidation)); + h.run_request( + &ok, + &ApplyEnv { + validation_snapshot: Some(snapshot), + ..env() + }, + ) + .unwrap(); + assert_eq!(h.revision(), 5); + assert_eq!( + h.record + .as_ref() + .unwrap() + .validation + .as_ref() + .unwrap() + .snapshot, + Some(snapshot) + ); + + h.run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetWriteFenced, 6)); + assert!(h.record.as_ref().unwrap().read_admission().is_open()); + assert!(!h.record.as_ref().unwrap().write_admission().is_open()); + + h.run(OP, CommandKind::EnableTargetWrites).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetWritable, 7)); + assert!(h.record.as_ref().unwrap().write_admission().is_open()); + } + + #[test] + fn validation_summary_is_bounded() { + let mut h = Harness::in_state(S::TargetValidating); + let request = h.request( + OP, + FenceCommand::RecordTargetValidation { + result: ValidationResult::Ok, + summary: "x".repeat(MAX_VALIDATION_SUMMARY_BYTES + 1), + }, + ); + let err = h.run_request(&request, &env()).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::InvalidArgument)); + } + + /// Section 2.8: nothing moves a published target back, for the owning operation. + #[test] + fn target_writable_is_irreversible() { + for kind in CommandKind::ALL { + let mut h = Harness::in_state(S::TargetWritable); + let before = h.record.clone(); + let result = h.run(OP, kind); + match kind { + CommandKind::EnableTargetWrites => { + assert_eq!(outcome_of(&result), O::AlreadyApplied) + } + _ => assert_eq!(outcome_of(&result), O::InvalidFenceTransition, "{kind}"), + } + assert_eq!(h.record, before, "{kind} changed a published target"); + } + + // Another operation cannot move it to a frozen, aborted or absent target state + // either; it can only start a new move with the namespace as a source. + for kind in CommandKind::ALL { + let mut h = Harness::in_state(S::TargetWritable); + let result = h.run(OTHER_OP, kind); + if kind == CommandKind::AcquireSourceWriteFence { + assert_eq!(state_after(&result), Some(S::SourceDraining)); + } else { + assert!(outcome_of(&result).is_error(), "{kind}: {result:?}"); + assert_eq!(h.state(), S::TargetWritable); + } + } + } + + #[test] + fn exact_replay_after_revision_advanced() { + let mut h = Harness::source(); + let acquire = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&acquire, &env()).unwrap(); + + // While draining, a replay resumes the same drain. + let d = h.decide(&acquire, &env()).unwrap(); + assert!(matches!(&d, Decision::Resume(r) if r.command_id == acquire.command_id)); + + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }); + h.run(OP, CommandKind::SetSourceReadFence).unwrap(); + h.complete(DrainCompletion::SourceReads); + assert_eq!(h.revision(), 4); + + // The original acquire's expectations (UNFENCED, 0) are long stale, but a replay is + // answered from its receipt before any revision check. + let d = h.decide(&acquire, &env()).unwrap(); + let Decision::Replay(receipt) = d else { + panic!("{d:?}") + }; + assert_eq!(receipt.outcome, O::Applied); + assert_eq!(receipt.state_after, S::SourceWriteFenced); + assert_eq!(receipt.revision_after, 2); + } + + #[test] + fn command_id_reuse_with_different_fingerprint_conflicts() { + let mut h = Harness::source(); + let acquire = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&acquire, &env()).unwrap(); + let before = (h.record.clone(), h.receipts.clone()); + + let mut reused = acquire.clone(); + reused.command = FenceCommand::ReleaseSourceWriteFence; + reused.expected_state = S::SourceDraining; + reused.expected_revision = 1; + let err = h.decide(&reused, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceCommandConflict); + + let mut reused = acquire.clone(); + reused.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: None, + }; + assert_eq!( + h.decide(&reused, &env()).unwrap_err().outcome(), + O::FenceCommandConflict + ); + + assert_eq!((h.record.clone(), h.receipts.clone()), before); + } + + #[test] + fn wrong_owner_is_refused() { + for state in RECORD_STATES { + if !state.is_durable() || state.is_operation_finished() { + continue; + } + for kind in CommandKind::ALL { + if kind == CommandKind::AdoptFence { + continue; + } + let mut h = Harness::in_state(state); + let before = h.record.clone(); + let result = h.run(OTHER_OP, kind); + // The owner is checked before anything about the command. + assert_eq!( + outcome_of(&result), + O::FenceOwnedByAnotherOperation, + "{state} {kind}" + ); + assert_eq!(h.record, before); + } + } + } + + #[test] + fn stale_revision_is_refused() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_revision -= 1; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_state = S::SourceDraining; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_revision += 1; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + } + + #[test] + fn role_mismatch() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let err = h.run(OP, CommandKind::SealTargetImport).unwrap_err(); + assert_eq!(err.outcome(), O::InvalidFenceTransition); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::in_state(S::TargetQuarantined); + let err = h.run(OP, CommandKind::SetSourceReadFence).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::source(); + let err = h.run(OP, CommandKind::EnableTargetWrites).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::target(); + let err = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + } + + /// The pure half of `acquire_race_single_owner`: two operations race to acquire the same + /// namespace; the store's serialised transactions mean the second decides against the + /// first's committed record and loses with a typed conflict. + #[test] + fn acquire_race_single_owner() { + let mut h = Harness::source(); + let a = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + let b = h.request(OTHER_OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&a, &env()).unwrap(); + let err = h.run_request(&b, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceOwnedByAnotherOperation); + assert_eq!(h.record.as_ref().unwrap().operation_id, OP); + } + + #[test] + fn acquire_preconditions() { + let mut h = Harness::source(); + let request = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + let err = h + .decide( + &request, + &ApplyEnv { + namespace_log_id: Some(Uuid::from_u128(0x11)), + ..env() + }, + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::NamespaceIdentityMismatch)); + + let err = h + .decide( + &request, + &ApplyEnv { + shared_schema: true, + ..env() + }, + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::SharedSchemaUnsupported)); + } + + #[test] + fn acquire_saves_and_release_restores_legacy_blocks() { + let saved = LegacyBlocks { + block_reads: false, + block_writes: true, + block_reason: Some("maintenance".into()), + }; + let mut h = Harness::source(); + let request = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request( + &request, + &ApplyEnv { + legacy_blocks: saved.clone(), + ..env() + }, + ) + .unwrap(); + let mirror = h.record.as_ref().unwrap().legacy_mirror(); + assert!(mirror.block_writes); + assert!(mirror + .block_reason + .unwrap() + .starts_with("namespace fence: SOURCE_DRAINING")); + + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!(h.record.as_ref().unwrap().legacy_mirror(), saved); + } + + #[test] + fn release_from_draining_is_a_precommit_rollback() { + let mut h = Harness::in_state(S::SourceDraining); + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::Released, 2)); + } + + #[test] + fn owner_joins_its_own_drain_with_a_new_command() { + let mut h = Harness::in_state(S::SourceDraining); + let d = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap(); + let Decision::Apply { record, receipt } = d else { + panic!() + }; + assert!(record.is_none()); + assert_eq!(receipt.outcome, O::Draining); + assert_eq!(h.revision(), 1); + + // The drain completes through the new command's receipt. + let record = h.record.clone().unwrap(); + let (next, final_receipt) = complete_drain( + &record, + &receipt, + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 3, + }, + }, + &env(), + ) + .unwrap(); + assert_eq!(next.state, S::SourceWriteFenced); + assert_eq!(final_receipt.command_id, receipt.command_id); + assert_eq!(final_receipt.outcome, O::Applied); + } + + #[test] + fn complete_drain_checks() { + let h = Harness::in_state(S::SourceDraining); + let record = h.record.clone().unwrap(); + let receipt = h.receipts[0].clone(); + + // Wrong kind of completion for the state. + assert!(complete_drain(&record, &receipt, DrainCompletion::SourceReads, &env()).is_err()); + assert!(complete_drain(&record, &receipt, DrainCompletion::TargetImport, &env()).is_err()); + + // A boundary from another log. + let err = complete_drain( + &record, + &receipt, + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: Uuid::from_u128(0x77), + frame_no: 1, + }, + }, + &env(), + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::NamespaceIdentityMismatch)); + + // A final receipt, or another operation's. + let mut final_receipt = receipt.clone(); + final_receipt.outcome = O::Applied; + let boundary = DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }; + assert!(complete_drain(&record, &final_receipt, boundary, &env()).is_err()); + let mut other = receipt.clone(); + other.operation_id = OTHER_OP; + assert!(complete_drain(&record, &other, boundary, &env()).is_err()); + + // Not draining any more. + let h = Harness::in_state(S::SourceWriteFenced); + let record = h.record.clone().unwrap(); + assert!(complete_drain(&record, &receipt, boundary, &env()).is_err()); + } + + #[test] + fn unavailable_refuses_everything_else() { + let marker = Harness::in_state(S::SourceWriteFenced).record.unwrap(); + for detail in [ + FenceDetail::CorruptRecord, + FenceDetail::UnsupportedFormatVersion, + FenceDetail::MetastoreBehindMarker, + FenceDetail::IncompleteTargetCreation, + FenceDetail::IndeterminateCommit, + ] { + for kind in CommandKind::ALL { + if kind == CommandKind::AdoptFence && detail == FenceDetail::MetastoreBehindMarker { + continue; + } + let current = CurrentFence::Unavailable { + detail, + marker: Some(&marker), + }; + let request = FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: OP, + command_id: Uuid::from_u128(0x5000), + expected_state: S::UnknownUnavailable, + expected_revision: marker.revision, + command: command(kind), + }; + let err = apply(current, None, &request, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceStateUnavailable, "{detail} {kind}"); + assert_eq!(err.detail(), Some(detail)); + } + } + } + + #[test] + fn incomplete_target_creation_is_completed_only_by_the_same_command() { + let mut h = Harness::target(); + let create = h.request(OP, command(CommandKind::CreateTargetQuarantined)); + let Decision::Apply { record, .. } = h.decide(&create, &env()).unwrap() else { + panic!() + }; + // The store writes this marker, then crashes before the metastore commit. + let marker = FenceMarker::for_record(record.as_ref().unwrap()); + let marker = FenceMarker::decode(&marker.encode()).unwrap().record; + let current = CurrentFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + marker: Some(&marker), + }; + + // Another command id, even from the same operation, cannot complete it. + let mut other = create.clone(); + other.command_id = Uuid::from_u128(0x6000); + let err = apply(current, None, &other, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceStateUnavailable); + + // The same command does, with the incarnation id the marker recorded. + let d = apply( + current, + None, + &create, + &ApplyEnv { + new_incarnation_id: Uuid::from_u128(0x7777), + ..env() + }, + ) + .unwrap(); + let Decision::Apply { + record: Some(record), + receipt, + } = d + else { + panic!() + }; + assert_eq!(record, marker); + assert_eq!(receipt.outcome, O::Applied); + } + + fn adopt_request(h: &mut Harness, args: AdoptArgs) -> FenceRequest { + h.request(OTHER_OP, FenceCommand::AdoptFence(args)) + } + + fn adopt_args() -> AdoptArgs { + match command(CommandKind::AdoptFence) { + FenceCommand::AdoptFence(args) => args, + _ => unreachable!(), + } + } + + fn authorised() -> ApplyEnv { + ApplyEnv { + adoption_authorised: true, + ..env() + } + } + + #[test] + fn adopt_requires_key_and_two_approvers() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let request = adopt_request(&mut h, adopt_args()); + let err = h.decide(&request, &env()).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::AdoptionNotAuthorised)); + + let bad = [ + AdoptArgs { + approvers: vec!["alice".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["alice".into(), " alice ".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["alice".into(), "".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["a".into(), "b".into(), "c".into()], + ..adopt_args() + }, + AdoptArgs { + incident_ref: " ".into(), + ..adopt_args() + }, + AdoptArgs { + reason: "".into(), + ..adopt_args() + }, + ]; + for args in bad { + let request = adopt_request(&mut h, args.clone()); + let err = h.decide(&request, &authorised()).unwrap_err(); + assert_eq!( + err.detail(), + Some(FenceDetail::AdoptionNotAuthorised), + "{args:?}" + ); + } + + let wrong_owner = AdoptArgs { + current_operation_id: Uuid::from_u128(0xdead), + ..adopt_args() + }; + let request = adopt_request(&mut h, wrong_owner); + assert_eq!( + h.decide(&request, &authorised()).unwrap_err().outcome(), + O::FenceOwnedByAnotherOperation + ); + } + + #[test] + fn adopt_keeps_gates_closed() { + for state in [ + S::SourceDraining, + S::SourceWriteFenced, + S::SourceReadDraining, + S::SourceReadFenced, + S::TargetQuarantined, + S::TargetImportDraining, + S::TargetValidating, + S::TargetWriteFenced, + ] { + let mut h = Harness::in_state(state); + let before = h.record.clone().unwrap(); + let request = adopt_request(&mut h, adopt_args()); + h.run_request(&request, &authorised()).unwrap(); + let after = h.record.clone().unwrap(); + assert_eq!(after.state, state); + assert_eq!(after.write_admission(), before.write_admission()); + assert_eq!(after.read_admission(), before.read_admission()); + assert_eq!(after.revision, before.revision + 1); + assert_eq!(after.operation_id, OTHER_OP); + assert_eq!(after.adoptions.len(), 1); + assert_eq!(after.adoptions[0].approvers, vec!["alice", "bob"]); + let receipt = h.receipts.last().unwrap(); + assert_eq!(receipt.adoption.as_ref().unwrap().previous_operation_id, OP); + + // The old owner is now locked out. + let result = h.run(OP, CommandKind::ReleaseSourceWriteFence); + assert_eq!(outcome_of(&result), O::FenceOwnedByAnotherOperation); + } + } + + #[test] + fn adopt_cannot_touch_finished_fences() { + for state in [S::TargetWritable, S::TargetAborted, S::Released] { + let mut h = Harness::in_state(state); + let request = adopt_request(&mut h, adopt_args()); + let err = h.decide(&request, &authorised()).unwrap_err(); + assert_eq!(err.outcome(), O::InvalidFenceTransition, "{state}"); + assert_eq!(err.detail(), Some(FenceDetail::OperationFinished)); + } + let mut h = Harness::source(); + let request = adopt_request(&mut h, adopt_args()); + assert_eq!( + h.decide(&request, &authorised()).unwrap_err().outcome(), + O::InvalidFenceTransition + ); + } + + #[test] + fn adopted_owner_can_finish_the_drain() { + let mut h = Harness::in_state(S::SourceDraining); + let request = adopt_request(&mut h, adopt_args()); + h.run_request(&request, &authorised()).unwrap(); + let d = h + .run(OTHER_OP, CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert!( + matches!(d, Decision::Apply { record: None, ref receipt } if receipt.outcome == O::Draining) + ); + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 5, + }, + }); + assert_eq!(h.state(), S::SourceWriteFenced); + assert_eq!(h.record.as_ref().unwrap().operation_id, OTHER_OP); + } + + #[test] + fn adopt_after_metastore_rollback_restores_marker_record() { + let marker = Harness::in_state(S::SourceReadFenced).record.unwrap(); + let current = CurrentFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(&marker), + }; + let request = FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: OTHER_OP, + command_id: Uuid::from_u128(0x8000), + expected_state: S::SourceReadFenced, + expected_revision: marker.revision, + command: FenceCommand::AdoptFence(adopt_args()), + }; + let d = apply(current, None, &request, &authorised()).unwrap(); + let Decision::Apply { + record: Some(record), + .. + } = d + else { + panic!() + }; + assert_eq!(record.state, S::SourceReadFenced); + assert_eq!(record.revision, marker.revision + 1); + assert_eq!(record.operation_id, OTHER_OP); + assert_eq!(record.frozen_boundary, marker.frozen_boundary); + } + + #[test] + fn revision_increases_by_one_per_applied_transition() { + let h = Harness::in_state(S::TargetWritable); + let mut receipts = h.receipts.clone(); + receipts.sort_by_key(|r| r.revision_after); + let mut last = 0; + for r in receipts { + if r.outcome == O::AlreadyApplied { + continue; + } + assert_eq!( + r.revision_after, + last + if r.command == CommandKind::SealTargetImport { + 2 + } else { + 1 + }, + "{r:?}" + ); + last = r.revision_after; + } + assert_eq!(h.revision(), 6); + } +} diff --git a/libsql-server/src/namespace/meta_store.rs b/libsql-server/src/namespace/meta_store.rs index 70b419ebe9..175fb24b55 100644 --- a/libsql-server/src/namespace/meta_store.rs +++ b/libsql-server/src/namespace/meta_store.rs @@ -1,6 +1,7 @@ #![allow(clippy::mutable_key_type)] -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use std::{collections::HashMap, fs::read_dir}; use bottomless::bottomless_wal::BottomlessWalWrapper; @@ -14,11 +15,13 @@ use libsql_sys::wal::{ }; use parking_lot::Mutex; use prost::Message; +use rusqlite::TransactionBehavior; use tokio::sync::oneshot; use tokio::sync::{ mpsc, watch::{self, Receiver, Sender}, }; +use uuid::Uuid; use crate::config::BottomlessConfig; use crate::connection::config::DatabaseConfig; @@ -28,6 +31,16 @@ use crate::{ config::MetaStoreConfig, connection::legacy::open_conn_active_checkpoint, error::Error, Result, }; +use super::fence::command::{FenceCommand, FenceRequest}; +use super::fence::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::fence::record::{ + CommandReceipt, NamespaceFenceRecord, ServerIdentity, ValidationSnapshot, +}; +use super::fence::state::OperationClass; +use super::fence::store::{ + self as fence_store, FenceStoreError, MarkerStatus, StoredFence, StoredReceipt, +}; +use super::fence::transition::{self, ApplyEnv, Decision, DrainCompletion}; use super::NamespaceName; type ChangeMsg = ( @@ -75,6 +88,28 @@ struct MetaStoreInner { conn: tokio::sync::Mutex, wal_manager: MetaStoreWalManager, db_kind: DatabaseKind, + /// `/dbs`, where namespace directories and their fence markers are. + dbs_path: PathBuf, + fence: FenceSettings, + /// Namespaces whose state could not be recovered at startup and that are therefore + /// `UNKNOWN_UNAVAILABLE` (`docs/NAMESPACE_FENCE.md` section 13.3): an undecodable config + /// row, a fence that cannot be established, or a marker the metastore has no trustworthy + /// record for. They are refused by lookups and by every config or lifecycle change, and + /// never default-created. A fence command that commits for the name takes it out. + recovered: Mutex>, +} + +/// How this metastore treats namespace fences (`docs/NAMESPACE_FENCE.md` section 13.1). +#[derive(Debug, Clone, Copy)] +struct FenceSettings { + /// The fence may be used: its tables exist and commands are accepted. + enabled: bool, + /// The fence tables exist, so fence state is loaded and enforced. + tables: bool, + /// Recovery fails closed (section 13.3): the flag is on, the fence tables exist, or a + /// namespace directory holds a marker. + fail_closed: bool, + receipt_retention: Duration, } fn setup_connection(conn: &rusqlite::Connection) -> Result<()> { @@ -187,12 +222,29 @@ impl MetaStoreInner { db_kind: DatabaseKind, ) -> Result { setup_connection(&conn)?; + if config.namespace_fence { + fence_store::create_tables(&conn)?; + } + let tables = fence_store::tables_exist(&conn)?; + let dbs_path = base_path.join("dbs"); + let marked = marked_namespaces(&dbs_path)?; + let fence = FenceSettings { + enabled: config.namespace_fence, + tables, + fail_closed: config.namespace_fence || tables || !marked.is_empty(), + receipt_retention: config + .namespace_fence_receipt_retention + .unwrap_or(fence_store::DEFAULT_RECEIPT_RETENTION), + }; let mut this = MetaStoreInner { configs: Default::default(), conn: conn.into(), wal_manager, db_kind, + dbs_path, + fence, + recovered: Default::default(), }; if config.allow_recover_from_fs { @@ -200,10 +252,75 @@ impl MetaStoreInner { } this.restore()?; + if this.fence.tables { + this.restore_fences()?; + } + this.register_marked(&marked)?; Ok(this) } + /// Register every namespace directory with a marker that the metastore has no trustworthy + /// fence record for as `UNKNOWN_UNAVAILABLE` (section 13.3): a metastore that was rebuilt + /// (`destroy_on_error`), recovered from the filesystem, restored from an older backup, or + /// that lost its fence tables, and a target whose creation was interrupted. + fn register_marked(&mut self, marked: &[NamespaceName]) -> Result<()> { + for ns in marked { + if self.recovered.get_mut().contains_key(ns) { + continue; + } + let known = self.configs.get_mut().contains_key(ns); + if self.fence.tables { + if known { + // `restore_fences` compared the marker with the record. + continue; + } + let stored = match fence_store::read_fence(self.conn.get_mut(), &self.dbs_path, ns) + { + Ok((stored, _)) => stored, + Err(FenceStoreError::Sqlite(e)) => return Err(e.into()), + Err(e) => StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be read: {e}"), + marker: None, + }, + }; + let (detail, reason, marker) = match stored { + StoredFence::Unavailable { + detail, + reason, + marker, + } => (detail, reason, marker), + other => ( + FenceDetail::CorruptRecord, + format!( + "the namespace has fence state {} but no usable config row", + other.state() + ), + other.record().cloned(), + ), + }; + mark_unavailable(self.recovered.get_mut(), ns.clone(), detail, reason, marker); + } else { + let (detail, marker) = match fence_store::read_marker(&self.dbs_path, ns)? { + None => continue, + Some(Ok(m)) => (FenceDetail::MetastoreBehindMarker, Some(m.record)), + Some(Err(_)) => (FenceDetail::CorruptRecord, None), + }; + let reason = "the namespace directory holds a fence marker but the metastore has \ + no fence tables (it was rebuilt, recovered or restored without them)" + .to_string(); + mark_unavailable(self.recovered.get_mut(), ns.clone(), detail, reason, marker); + } + } + Ok(()) + } + + /// The fence error for a namespace registered as unavailable at startup. + fn recovery_denial(&self, namespace: &NamespaceName) -> Option { + self.recovered.lock().get(namespace).map(unavailable_error) + } + fn maybe_recover_from_fs(&mut self, base_path: &Path) -> Result<()> { let count = self.conn @@ -226,6 +343,16 @@ impl MetaStoreInner { let config_path = entry.path().join("config.json"); let name = NamespaceName::from_string(entry.file_name().to_str().unwrap().to_string())?; + if entry + .path() + .join(fence_store::MARKER_FILE_NAME) + .try_exists()? + { + // A fenced namespace is never recovered with a guessed config; it is + // registered as unavailable below (section 13.3). + tracing::warn!("not recovering fenced namespace `{name}` from the filesystem"); + continue; + } let config = if config_path.try_exists()? { let config_bytes = std::fs::read(&config_path)?; serde_json::from_slice(&config_bytes)? @@ -250,10 +377,10 @@ impl MetaStoreInner { fn restore(&mut self) -> Result<()> { tracing::info!("restoring meta store"); - let mut stmt = self - .conn - .get_mut() - .prepare("SELECT namespace, config FROM namespace_configs")?; + let fence = self.fence; + let conn: &rusqlite::Connection = self.conn.get_mut(); + let mut unavailable = Vec::new(); + let mut stmt = conn.prepare("SELECT namespace, config FROM namespace_configs")?; let rows = stmt.query(())?.mapped(|r| { let ns = r.get::<_, String>(0)?; @@ -265,9 +392,19 @@ impl MetaStoreInner { for row in rows { match row { Ok((k, v)) => { - let ns = match NamespaceName::from_string(k) { + let ns = match NamespaceName::from_string(k.clone()) { Ok(ns) => ns, Err(e) => { + // A name nothing can address cannot be served or default-created, + // so a legacy row is skipped as before. A fenced one is an operator + // problem: its fence could not be enforced or inspected. + if fence.tables && fence_store::stored_revision_raw(conn, &k)?.is_some() + { + return Err(Error::Internal(format!( + "the metastore holds a namespace fence for `{k}`, which is not \ + a valid namespace name; refusing to start" + ))); + } tracing::warn!("unable to convert namespace name: {}", e); continue; } @@ -275,6 +412,11 @@ impl MetaStoreInner { let config = match metadata::DatabaseConfig::decode(&v[..]) { Ok(c) => Arc::new(DatabaseConfig::from(&c)), + Err(e) if fence.fail_closed => { + unavailable + .push((ns, format!("the config row cannot be decoded: {e}"))); + continue; + } Err(e) => { tracing::warn!("unable to convert config: {}", e); continue; @@ -297,10 +439,184 @@ impl MetaStoreInner { } } + drop(stmt); + for (ns, reason) in unavailable { + mark_unavailable( + self.recovered.get_mut(), + ns, + FenceDetail::CorruptRecord, + reason, + None, + ); + } + tracing::info!("meta store restore completed"); Ok(()) } + + /// Load every namespace's fence after the configs (section 5.6). The stored config row of + /// a fenced namespace carries the legacy mirror of the fence in its `block_*` fields + /// (section 13.2); the in-memory config is the namespace's own configuration, so those + /// fields are put back to the values the record saved. A marker that fell behind its + /// record is rewritten. A namespace whose fence cannot be established is logged and keeps + /// its stored config, mirror included. + fn restore_fences(&mut self) -> Result<()> { + let namespaces: Vec = self.configs.get_mut().keys().cloned().collect(); + let conn = self.conn.get_mut(); + let mut fenced = 0usize; + for ns in namespaces { + let (stored, marker) = match fence_store::read_fence(conn, &self.dbs_path, &ns) { + Ok(r) => r, + Err(FenceStoreError::Sqlite(e)) => return Err(e.into()), + Err(e) => { + fenced += 1; + mark_unavailable( + self.recovered.get_mut(), + ns, + FenceDetail::CorruptRecord, + format!("the namespace fence cannot be established: {e}"), + None, + ); + continue; + } + }; + match &stored { + StoredFence::None { .. } => continue, + StoredFence::Record(record) => { + fenced += 1; + if marker == MarkerStatus::Stale { + if let Err(e) = fence_store::write_marker(&self.dbs_path, record) { + tracing::error!(namespace = %ns, "failed to rewrite fence marker: {e}"); + } + } + let sender = self.configs.get_mut().get_mut(&ns).expect("listed above"); + let config = sender.borrow().config.clone(); + let config = fence_store::with_legacy_blocks(&config, &record.legacy_blocks); + sender.send_modify(|c| c.config = Arc::new(config)); + } + StoredFence::Unavailable { + detail, + reason, + marker, + } => { + fenced += 1; + mark_unavailable( + self.recovered.get_mut(), + ns, + *detail, + reason.clone(), + marker.clone(), + ); + } + } + } + tracing::info!("loaded {fenced} namespace fence(s)"); + Ok(()) + } +} + +fn mark_unavailable( + recovered: &mut HashMap, + namespace: NamespaceName, + detail: FenceDetail, + reason: String, + marker: Option, +) { + tracing::error!( + namespace = %namespace, + %detail, + "namespace is UNKNOWN_UNAVAILABLE: {reason}" + ); + recovered.insert( + namespace, + StoredFence::Unavailable { + detail, + reason, + marker, + }, + ); +} + +/// The namespaces under `dbs_path` whose directory holds a fence marker. A marker in a +/// directory that is not a valid namespace name stops startup: the fence it records could be +/// neither enforced nor inspected. +fn marked_namespaces(dbs_path: &Path) -> Result> { + fence_store::scan_markers(dbs_path)? + .into_iter() + .map(|m| { + m.map_err(|raw| { + Error::Internal(format!( + "namespace directory `{raw}` holds a fence marker but is not a valid \ + namespace name; refusing to start" + )) + }) + }) + .collect() +} + +/// A namespace's fence as the metastore holds it now, for a metastore with the fence tables: +/// the live record and marker, unless they read as established while startup could not +/// recover the namespace (an undecodable config row, for instance), which stays unavailable. +fn established_fence( + inner: &MetaStoreInner, + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> std::result::Result { + let (live, _) = fence_store::read_fence(conn, &inner.dbs_path, namespace)?; + if matches!(live, StoredFence::Unavailable { .. }) { + return Ok(live); + } + Ok(inner + .recovered + .lock() + .get(namespace) + .cloned() + .unwrap_or(live)) +} + +/// The error returned for a namespace whose fence state is `UNKNOWN_UNAVAILABLE`. +fn unavailable_error(stored: &StoredFence) -> FenceError { + stored + .permits(OperationClass::NormalRead) + .err() + .unwrap_or_else(|| { + FenceError::new( + FenceOutcome::FenceStateUnavailable, + "the namespace's fence state cannot be established", + ) + }) +} + +/// Why a name that has no config must not be created: its directory holds a marker, so it is a +/// target being created or a namespace the metastore lost (section 13.3). +fn marker_denial(dbs_path: &Path, namespace: &NamespaceName) -> Result> { + Ok(match fence_store::read_marker(dbs_path, namespace)? { + None => None, + Some(Ok(m)) => { + let revision = m.record.revision; + Some( + StoredFence::Record(m.record) + .permits(OperationClass::Lifecycle) + .err() + .unwrap_or_else(|| { + unavailable_error(&StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the namespace directory holds a fence marker (revision \ + {revision}) but the metastore has no config for it" + ), + marker: None, + }) + }), + ) + } + Some(Err(e)) => Some(unavailable_error(&StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be decoded: {e}"), + marker: None, + })), + }) } /// Handles config change updates by inserting them into the database and in-memory @@ -313,6 +629,11 @@ fn process(msg: ChangeMsg, inner: Arc) { } else { Ok(()) }; + // A config that was not persisted is not published. + if ret.is_err() { + let _ = ret_chan.send(ret); + return; + } let mut configs = inner.configs.blocking_lock(); if let Some(config_watch) = configs.get_mut(&namespace) { let new_version = config_watch.borrow().version.wrapping_add(1); @@ -349,22 +670,25 @@ fn try_process( namespace: &NamespaceName, config: &DatabaseConfig, ) -> Result<()> { - let config_encoded = metadata::DatabaseConfig::from(&*config).encode_to_vec(); - + if let Some(e) = inner.recovery_denial(namespace) { + return Err(e.into()); + } let mut conn = inner.conn.blocking_lock(); + // `BEGIN IMMEDIATE`: the write lock is what serialises this write with fence transitions + // (docs/NAMESPACE_FENCE.md section 5.4), including those of other metastore connections. + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if inner.fence.tables { + let (stored, _) = + fence_store::read_fence(&tx, &inner.dbs_path, namespace).map_err(fence_store_error)?; + stored.permits(OperationClass::Lifecycle)?; + } if let Some(schema) = config.shared_schema_name.as_ref() { - let tx = conn.transaction()?; if inner.db_kind.is_primary() { - if let Some(ref schema) = config.shared_schema_name { - if crate::schema::db::has_pending_migration_jobs(&tx, schema)? { - return Err(crate::Error::PendingMigrationOnSchema(schema.clone())); - } + if crate::schema::db::has_pending_migration_jobs(&tx, schema)? { + return Err(crate::Error::PendingMigrationOnSchema(schema.clone())); } } - tx.execute( - "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) ON CONFLICT(namespace) DO UPDATE SET config=excluded.config", - rusqlite::params![namespace.as_str(), config_encoded], - )?; + fence_store::write_config_row(&tx, namespace, config)?; tx.execute( "DELETE FROM shared_schema_links WHERE namespace = ?", rusqlite::params![namespace.as_str()], @@ -373,13 +697,10 @@ fn try_process( "INSERT OR REPLACE INTO shared_schema_links (shared_schema_name, namespace) VALUES (?1, ?2)", rusqlite::params![schema.as_str(), namespace.as_str()], )?; - tx.commit()?; } else { - conn.execute( - "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) ON CONFLICT(namespace) DO UPDATE SET config=excluded.config", - rusqlite::params![namespace.as_str(), config_encoded], - )?; + fence_store::write_config_row(&tx, namespace, config)?; } + tx.commit()?; if let Err(e) = checkpoint(&conn) { tracing::warn!("failed to checkpoint metastore: {e}"); @@ -388,11 +709,339 @@ fn try_process( Ok(()) } +fn fence_store_error(e: FenceStoreError) -> Error { + match e { + FenceStoreError::Fence(e) => Error::NamespaceFence(e), + FenceStoreError::Sqlite(e) => Error::RusqliteError(e), + FenceStoreError::Io(e) => Error::IOError(e), + } +} + fn checkpoint(conn: &rusqlite::Connection) -> Result<()> { conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", (), |_| Ok(()))?; Ok(()) } +/// Facts about the server and the live namespace that a fence command needs and the metastore +/// does not hold (see [`ApplyEnv`]). The store adds what it reads inside the transaction. +#[derive(Debug, Clone)] +pub struct FenceContext { + pub server: ServerIdentity, + /// Wall-clock time in milliseconds since the Unix epoch. + pub now_ms: i64, + /// The namespace's current replication log id, if it exists and has one. + pub namespace_log_id: Option, + /// A fresh id for `CreateTargetQuarantined`. + pub new_incarnation_id: Uuid, + /// Whether the request carried the configured adoption key. + pub adoption_authorised: bool, + /// What the server observed of a target, for `RecordTargetValidation`. + pub validation_snapshot: Option, +} + +impl FenceContext { + /// A context for `server` at the current time with a fresh incarnation id. + pub fn now(server: ServerIdentity, namespace_log_id: Option) -> Self { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)); + Self { + server, + now_ms, + namespace_log_id, + new_incarnation_id: Uuid::new_v4(), + adoption_authorised: false, + validation_snapshot: None, + } + } +} + +/// How a fence command was answered. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FenceCommitKind { + /// The command had been applied before; nothing was written. + Replayed, + /// The command started a drain that is still to be completed; nothing was written and the + /// controller resumes the drain. + Resumed, + /// The command's receipt (and, where it changed, the record) was committed. + Committed, +} + +/// The committed result of a fence command. +#[derive(Debug, Clone)] +pub struct FenceCommit { + pub kind: FenceCommitKind, + pub receipt: CommandReceipt, + /// The namespace's fence record after the command (for a replay, as it is now). + pub record: Option, + /// For a `CreateTargetQuarantined` that was committed, the namespace config it created. + /// It is not in the in-memory config map yet: the caller installs the target's gate first + /// and then publishes it. + pub created_config: Option>, +} + +/// A namespace's fence as read by `inspect_fence`. +#[derive(Debug, Clone)] +pub struct FenceInspection { + pub fence: StoredFence, + pub receipts: Vec, +} + +fn fence_disabled() -> FenceError { + FenceError::new( + FenceOutcome::FencePreconditionFailed, + "namespace fences are not enabled on this server", + ) + .with_detail(FenceDetail::FenceDisabled) +} + +fn not_primary() -> FenceError { + FenceError::new( + FenceOutcome::FencePreconditionFailed, + "namespace fences are only changed on a primary", + ) + .with_detail(FenceDetail::NotPrimary) +} + +fn unavailable_receipt(e: impl std::fmt::Display) -> FenceError { + FenceError::new( + FenceOutcome::FenceStateUnavailable, + format!("the stored receipt for this command cannot be read: {e}"), + ) + .with_detail(FenceDetail::CorruptRecord) +} + +/// The `ApplyEnv` for a command: the caller's context plus what the transaction read. +fn apply_env( + ctx: &FenceContext, + stored: &StoredFence, + config: Option<&DatabaseConfig>, +) -> ApplyEnv { + ApplyEnv { + now_ms: ctx.now_ms, + server: ctx.server.clone(), + namespace_log_id: ctx.namespace_log_id, + shared_schema: config.is_some_and(|c| c.is_shared_schema || c.shared_schema_name.is_some()), + // The namespace's own values: a stored record's saved values while it is in force, + // otherwise the config row as stored. + legacy_blocks: match stored { + StoredFence::Record(r) if !r.state.is_operation_finished() => r.legacy_blocks.clone(), + _ => config + .map(fence_store::legacy_blocks_of) + .unwrap_or_default(), + }, + new_incarnation_id: ctx.new_incarnation_id, + adoption_authorised: ctx.adoption_authorised, + validation_snapshot: ctx.validation_snapshot, + } +} + +fn apply_fence_command( + inner: &MetaStoreInner, + request: &FenceRequest, + ctx: &FenceContext, +) -> std::result::Result { + if !inner.fence.enabled || !inner.fence.tables { + return Err(fence_disabled().into()); + } + if !inner.db_kind.is_primary() { + return Err(not_primary().into()); + } + let ns = &request.namespace; + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let (stored, marker) = fence_store::read_fence(&tx, &inner.dbs_path, ns)?; + let existing = + match fence_store::read_receipt(&tx, ns, request.operation_id, request.command_id)? { + None => None, + Some(Ok(r)) => Some(r), + Some(Err(e)) => return Err(unavailable_receipt(e).into()), + }; + let config = fence_store::read_config_row(&tx, ns)?; + let env = apply_env(ctx, &stored, config.as_ref()); + + let decision = transition::apply(stored.as_current(), existing.as_ref(), request, &env)?; + let (record, receipt) = match decision { + Decision::Replay(receipt) | Decision::Resume(receipt) => { + let kind = if receipt.is_final() { + FenceCommitKind::Replayed + } else { + FenceCommitKind::Resumed + }; + return Ok(FenceCommit { + kind, + receipt, + record: stored.record().cloned(), + created_config: None, + }); + } + Decision::Apply { record, receipt } => (record, receipt), + }; + + let mut created_config = None; + if let Some(next) = &record { + let previous = fence_store::stored_revision(&tx, ns)?; + if let FenceCommand::CreateTargetQuarantined { config: target } = &request.command { + // Section 10.1: the marker first, then the config row, the record and the receipt + // in one transaction. A crash in between leaves a marker without rows, which only a + // replay of this command completes. + let logical = fence_store::target_database_config(target)?; + fence_store::write_marker(&inner.dbs_path, next)?; + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(&logical, &next.legacy_mirror()), + )?; + created_config = Some(Arc::new(logical)); + } else { + let Some(config) = &config else { + return Err(FenceError::new( + FenceOutcome::FenceStateUnavailable, + "the fenced namespace has no config row", + ) + .with_detail(FenceDetail::CorruptRecord) + .into()); + }; + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(config, &next.legacy_mirror()), + )?; + } + fence_store::write_record(&tx, next, previous)?; + } + fence_store::write_receipt(&tx, &receipt)?; + let owner = record + .as_ref() + .or(stored.record()) + .map_or(request.operation_id, |r| r.operation_id); + fence_store::prune_receipts(&tx, ns, owner, ctx.now_ms, inner.fence.receipt_retention)?; + tx.commit()?; + // The command established the fence from the durable state; whatever startup could not + // recover about this name is settled. + inner.recovered.lock().remove(ns); + + let current = record.or_else(|| stored.record().cloned()); + after_fence_commit( + inner, + &conn, + current.as_ref(), + record_changed(¤t, &stored, marker), + ); + + Ok(FenceCommit { + kind: FenceCommitKind::Committed, + receipt, + record: current, + created_config, + }) +} + +/// Whether the marker has to be written after a commit: the record changed, or it had fallen +/// behind. +fn record_changed( + current: &Option, + stored: &StoredFence, + marker: MarkerStatus, +) -> bool { + current.as_ref() != stored.record() || marker == MarkerStatus::Stale +} + +fn after_fence_commit( + inner: &MetaStoreInner, + conn: &rusqlite::Connection, + record: Option<&NamespaceFenceRecord>, + write_marker: bool, +) { + if let (Some(record), true) = (record, write_marker) { + // The metastore is authoritative; a marker that is missing or behind is repaired on + // the next load (section 5.6). + if let Err(e) = fence_store::write_marker(&inner.dbs_path, record) { + tracing::error!(namespace = %record.namespace, "failed to write fence marker: {e}"); + } + } + if let Err(e) = checkpoint(conn) { + tracing::warn!("failed to checkpoint metastore: {e}"); + } +} + +fn complete_fence_drain( + inner: &MetaStoreInner, + ns: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, + completion: DrainCompletion, + ctx: &FenceContext, +) -> std::result::Result { + if !inner.fence.tables { + return Err(fence_disabled().into()); + } + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let (stored, _) = fence_store::read_fence(&tx, &inner.dbs_path, ns)?; + let record = match &stored { + StoredFence::Record(r) => r.clone(), + other => { + return Err(other + .permits(OperationClass::NormalWrite) + .err() + .unwrap_or_else(|| { + FenceError::new( + FenceOutcome::InvalidFenceTransition, + "the namespace has no fence record", + ) + }) + .into()) + } + }; + let receipt = match fence_store::read_receipt(&tx, ns, operation_id, command_id)? { + Some(Ok(r)) => r, + Some(Err(e)) => return Err(unavailable_receipt(e).into()), + None => { + return Err(FenceError::new( + FenceOutcome::InvalidFenceTransition, + format!("command {command_id} of operation {operation_id} has no receipt"), + ) + .into()) + } + }; + if receipt.is_final() { + return Ok(FenceCommit { + kind: FenceCommitKind::Replayed, + receipt, + record: Some(record), + created_config: None, + }); + } + + let config = fence_store::read_config_row(&tx, ns)?; + let env = apply_env(ctx, &stored, config.as_ref()); + let (next, final_receipt) = transition::complete_drain(&record, &receipt, completion, &env)?; + if let Some(config) = &config { + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(config, &next.legacy_mirror()), + )?; + } + fence_store::write_record(&tx, &next, fence_store::stored_revision(&tx, ns)?)?; + fence_store::write_receipt(&tx, &final_receipt)?; + tx.commit()?; + inner.recovered.lock().remove(ns); + + after_fence_commit(inner, &conn, Some(&next), true); + + Ok(FenceCommit { + kind: FenceCommitKind::Committed, + receipt: final_receipt, + record: Some(next), + created_config: None, + }) +} + impl MetaStore { #[tracing::instrument(skip(config, base_path, conn, wal_manager))] pub async fn new( @@ -420,12 +1069,36 @@ impl MetaStore { if destroy_on_error { let db_path = base_path.join("metastore"); - tracing::info!( - "meta store set to destroy on restore error, removing metastore db path folder ({:?})", db_path - ); + // With fences in use the broken metastore may hold the only record of a + // fence, so it is kept aside for the operator rather than deleted, and the + // rebuilt metastore registers every marked namespace as unavailable + // (section 13.3). + let keep = config.namespace_fence + || marked_namespaces(&base_path.join("dbs")) + .map_or(true, |marked| !marked.is_empty()); + if keep { + let millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + let aside = base_path.join(format!("metastore.broken-{millis}")); + tracing::error!( + "meta store failed to restore ({e}); moving it aside to {aside:?} \ + and rebuilding it" + ); + if let Err(rename) = std::fs::rename(&db_path, &aside) { + tracing::error!( + "failed to move the metastore aside ({rename}); not destroying it" + ); + return Err(e); + } + } else { + tracing::info!( + "meta store set to destroy on restore error, removing metastore db path folder ({:?})", db_path + ); - if let Err(e) = std::fs::remove_dir_all(&db_path) { - tracing::error!("failed to remove base path({:?}): {}", &db_path, e); + if let Err(e) = std::fs::remove_dir_all(&db_path) { + tracing::error!("failed to remove base path({:?}): {}", &db_path, e); + } } if let Err(e) = std::fs::create_dir_all(&db_path) { @@ -486,11 +1159,38 @@ impl MetaStore { Ok(Self { changes_tx, inner }) } - pub async fn handle(&self, namespace: NamespaceName) -> MetaStoreHandle { + /// The handle of an existing namespace, without creating one (section 13.3). `Ok(None)` + /// when the namespace does not exist; a fence error when its state could not be recovered. + /// Every path that only reads or serves a namespace uses this. + pub async fn lookup(&self, namespace: &NamespaceName) -> Result> { + if let Some(e) = self.inner.recovery_denial(namespace) { + return Err(e.into()); + } + let configs = self.inner.configs.lock().await; + Ok(configs.get(namespace).map(|sender| MetaStoreHandle { + namespace: namespace.clone(), + inner: HandleState::External(self.changes_tx.clone(), sender.subscribe()), + })) + } + + /// The handle of `namespace`, creating an empty in-memory entry when it does not exist. + /// Only paths that create a namespace (create, fork destination, reset, lazy creation) use + /// this. It refuses a namespace whose state could not be recovered, and a name without a + /// config whose directory holds a fence marker (a target being created, or a namespace the + /// metastore lost): creating either would publish a default config where a fence belongs. + pub async fn handle(&self, namespace: NamespaceName) -> Result { tracing::debug!("getting meta store handle"); + if let Some(e) = self.inner.recovery_denial(&namespace) { + return Err(e.into()); + } let change_tx = self.changes_tx.clone(); let mut configs = self.inner.configs.lock().await; + if !configs.contains_key(&namespace) { + if let Some(e) = marker_denial(&self.inner.dbs_path, &namespace)? { + return Err(e.into()); + } + } let sender = configs.entry(namespace.clone()).or_insert_with(|| { // TODO(lucio): if no entry exists we need to ensure we send the update to // the bg channel. @@ -502,14 +1202,17 @@ impl MetaStore { tracing::debug!("meta handle subscribed"); - MetaStoreHandle { + Ok(MetaStoreHandle { namespace, inner: HandleState::External(change_tx, rx), - } + }) } pub fn remove(&self, namespace: NamespaceName) -> Result>> { tracing::debug!("removing namespace `{}` from meta store", namespace); + if let Some(e) = self.inner.recovery_denial(&namespace) { + return Err(e.into()); + } // "configs" lock can be used in both async and sync contexts while "conn" lock always used // in blocking context @@ -523,7 +1226,26 @@ impl MetaStore { let r = if let Some(sender) = configs.get(&namespace) { tracing::debug!("removed namespace `{}` from meta store", namespace); let config = sender.borrow().clone(); - let tx = conn.transaction()?; + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if self.inner.fence.tables { + let (stored, _) = fence_store::read_fence(&tx, &self.inner.dbs_path, &namespace) + .map_err(fence_store_error)?; + stored.permits(OperationClass::Lifecycle)?; + if !matches!(stored, StoredFence::None { .. }) { + // The marker goes before the commit: a crash in between leaves a record + // without a marker, which is repaired on load, rather than a marker + // without a record, which would make the name unavailable. + fence_store::remove_marker(&self.inner.dbs_path, &namespace)?; + let receipts = fence_store::delete_fence(&tx, &namespace)?; + tracing::info!( + namespace = %namespace, + state = %stored.state(), + revision = stored.revision(), + receipts, + "removing namespace fence with its namespace" + ); + } + } if config.config.is_shared_schema { if crate::schema::db::schema_has_linked_dbs(&tx, &namespace)? { return Err(crate::Error::HasLinkedDbs(namespace.clone())); @@ -562,6 +1284,133 @@ impl MetaStore { self.inner.configs.lock().await.contains_key(namespace) } + /// Whether namespace fences may be used on this server. + pub fn fence_enabled(&self) -> bool { + self.inner.fence.enabled + } + + /// Whether this metastore holds fence state, so fences are loaded and enforced. + pub fn fence_enforced(&self) -> bool { + self.inner.fence.tables + } + + /// Run one fence command as a compare-and-swap in a single metastore transaction + /// (`docs/NAMESPACE_FENCE.md` sections 5.3 and 5.4). Nothing is published here: the + /// caller publishes the result only after this returns, which is after the commit. + /// + /// A fence outcome that is an error (`FENCE_REVISION_MISMATCH`, …) is returned as + /// [`Error::NamespaceFence`] and nothing is written. + pub async fn apply_fence_command( + &self, + request: FenceRequest, + ctx: FenceContext, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || apply_fence_command(&inner, &request, &ctx)) + .await? + .map_err(fence_store_error) + } + + /// Finish the drain that the owning operation's `DRAINING` receipt + /// `(operation_id, command_id)` started, once the controller has proven `completion`. The + /// final receipt replaces the `DRAINING` one. If the drain was already completed, the + /// final receipt is returned as a replay. + pub async fn complete_fence_drain( + &self, + namespace: NamespaceName, + operation_id: Uuid, + command_id: Uuid, + completion: DrainCompletion, + ctx: FenceContext, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || { + complete_fence_drain( + &inner, + &namespace, + operation_id, + command_id, + completion, + &ctx, + ) + }) + .await? + .map_err(fence_store_error) + } + + /// Read a namespace's fence and all of its receipts (`InspectFence`). Never writes. + pub async fn inspect_fence(&self, namespace: NamespaceName) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { + let mut conn = inner.conn.blocking_lock(); + if !inner.fence.tables { + let recovered = inner.recovered.lock().get(&namespace).cloned(); + let fence = match recovered { + Some(fence) => fence, + None => { + let tx = conn.transaction()?; + StoredFence::None { + namespace_exists: fence_store::read_config_row(&tx, &namespace)? + .is_some(), + } + } + }; + return Ok(FenceInspection { + fence, + receipts: Vec::new(), + }); + } + let tx = conn.transaction()?; + let fence = established_fence(&inner, &tx, &namespace)?; + let receipts = fence_store::read_receipts(&tx, &namespace)?; + Ok(FenceInspection { fence, receipts }) + }) + .await? + .map_err(fence_store_error) + } + + /// Every namespace with fence state, and that state. Namespaces without a record or a + /// marker are left out. + pub async fn load_fences(&self) -> Result> { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { + let recovered: Vec<(NamespaceName, StoredFence)> = inner + .recovered + .lock() + .iter() + .map(|(ns, fence)| (ns.clone(), fence.clone())) + .collect(); + if !inner.fence.tables { + return Ok(recovered); + } + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction()?; + let mut names: Vec = { + let mut stmt = tx.prepare("SELECT namespace FROM namespace_configs")?; + let rows = stmt.query_map((), |r| r.get::<_, String>(0))?; + rows.collect::>>()? + .into_iter() + .filter_map(|name| NamespaceName::from_string(name).ok()) + .collect() + }; + for (ns, _) in recovered { + if !names.contains(&ns) { + names.push(ns); + } + } + let mut out = Vec::new(); + for ns in names { + let fence = established_fence(&inner, &tx, &ns)?; + if !matches!(fence, StoredFence::None { .. }) { + out.push((ns, fence)); + } + } + Ok(out) + }) + .await? + .map_err(fence_store_error) + } + pub(crate) async fn shutdown(&self) -> crate::Result<()> { let replicator = self.inner.wal_manager.wrapper().as_ref(); @@ -729,3 +1578,1219 @@ impl MetaStoreHandle { &self.namespace } } + +#[cfg(test)] +mod fence_tests { + use std::path::Path; + + use tempfile::tempdir; + + use super::*; + use crate::namespace::fence::command::TargetConfig; + use crate::namespace::fence::record::{FenceMarker, FrozenBoundary}; + use crate::namespace::fence::state::FenceState; + + const LOG: Uuid = Uuid::from_u128(0x10); + const INCARNATION: Uuid = Uuid::from_u128(0x20); + const OP: Uuid = Uuid::from_u128(0xa); + const OTHER_OP: Uuid = Uuid::from_u128(0xb); + + async fn open_with(dir: &Path, config: MetaStoreConfig) -> MetaStore { + let (maker, manager) = metastore_connection_maker(None, dir).await.unwrap(); + let conn = maker().unwrap(); + MetaStore::new(config, dir, conn, manager, DatabaseKind::Primary) + .await + .unwrap() + } + + async fn open(dir: &Path, fence: bool) -> MetaStore { + open_with( + dir, + MetaStoreConfig { + namespace_fence: fence, + ..Default::default() + }, + ) + .await + } + + /// A second, independent connection to the same metastore database (like the schema + /// scheduler's). + async fn raw(dir: &Path) -> MetaStoreConnection { + let (maker, _) = metastore_connection_maker(None, dir).await.unwrap(); + maker().unwrap() + } + + fn raw_config(conn: &rusqlite::Connection, ns: &str) -> DatabaseConfig { + fence_store::read_config_row(conn, &NamespaceName::from(ns.to_string().leak() as &str)) + .unwrap() + .unwrap() + } + + fn ctx(now_ms: i64) -> FenceContext { + FenceContext { + server: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(0x99), + }, + now_ms, + namespace_log_id: Some(LOG), + new_incarnation_id: INCARNATION, + adoption_authorised: false, + validation_snapshot: None, + } + } + + fn request( + ns: &'static str, + op: Uuid, + command_id: u128, + expected_state: FenceState, + expected_revision: u64, + command: FenceCommand, + ) -> FenceRequest { + FenceRequest { + namespace: ns.into(), + operation_id: op, + command_id: Uuid::from_u128(command_id), + expected_state, + expected_revision, + command, + } + } + + fn acquire(ns: &'static str, op: Uuid, command_id: u128) -> FenceRequest { + request( + ns, + op, + command_id, + FenceState::Unfenced, + 0, + FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: None, + }, + ) + } + + fn outcome_of(r: Result) -> FenceOutcome { + match r { + Ok(c) => c.receipt.outcome, + Err(Error::NamespaceFence(e)) => e.outcome(), + Err(e) => panic!("unexpected error: {e}"), + } + } + + fn fence_error(r: Result) -> FenceError { + match r { + Err(Error::NamespaceFence(e)) => e, + other => panic!("expected a fence error, got {other:?}"), + } + } + + async fn create_namespace(store: &MetaStore, ns: &'static str) -> MetaStoreHandle { + let handle = store.handle(ns.into()).await.unwrap(); + handle + .store(DatabaseConfig { + max_db_pages: 1234, + block_reason: Some("pre-fence".into()), + ..Default::default() + }) + .await + .unwrap(); + handle + } + + fn remove_blocking(store: &MetaStore, ns: &'static str) -> Result>> { + let store = store.clone(); + std::thread::spawn(move || store.remove(ns.into())) + .join() + .unwrap() + } + + #[tokio::test] + async fn fence_cas_persists_across_restart() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + + let commit = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + assert_eq!(commit.receipt.outcome, FenceOutcome::Draining); + let record = commit.record.unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::SourceDraining, 1) + ); + + // The stored row carries the legacy mirror; the in-memory config is untouched. + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "db"); + assert!(row.block_writes && !row.block_reads); + assert!(row + .block_reason + .unwrap() + .starts_with("namespace fence: SOURCE_DRAINING")); + assert!(!handle.get().block_writes); + + let boundary = FrozenBoundary { + log_id: LOG, + frame_no: 42, + }; + let commit = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { boundary }, + ctx(2_000), + ) + .await + .unwrap(); + assert_eq!(commit.receipt.outcome, FenceOutcome::Applied); + let record = commit.record.unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::SourceWriteFenced, 2) + ); + // Completing again is a replay of the final answer. + let again = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { boundary }, + ctx(2_500), + ) + .await + .unwrap(); + assert_eq!(again.kind, FenceCommitKind::Replayed); + + let marker = fence_store::read_marker(&dir.path().join("dbs"), &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker, FenceMarker::for_record(&record)); + + drop(handle); + drop(store); + + // Restart. + let store = open(dir.path(), true).await; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(inspection.fence, StoredFence::Record(record.clone())); + assert_eq!(inspection.fence.revision(), 2); + assert_eq!(record.frozen_boundary, Some(boundary)); + assert_eq!(inspection.receipts.len(), 1); + let receipt = inspection.receipts[0].receipt.clone().unwrap(); + assert_eq!( + (receipt.outcome, receipt.revision_after), + (FenceOutcome::Applied, 2) + ); + + // The in-memory config is the namespace's own, not the mirror. + let handle = store.handle("db".into()).await.unwrap(); + let config = handle.get(); + assert!(!config.block_writes && !config.block_reads); + assert_eq!(config.block_reason.as_deref(), Some("pre-fence")); + assert_eq!(config.max_db_pages, 1234); + + // The lost response of the first command is answered from its receipt, even though + // the revision has advanced. + let replay = store + .apply_fence_command(acquire("db", OP, 1), ctx(3_000)) + .await + .unwrap(); + assert_eq!(replay.kind, FenceCommitKind::Replayed); + assert_eq!(replay.receipt, receipt); + + let fences = store.load_fences().await.unwrap(); + assert_eq!( + fences, + vec![("db".into(), StoredFence::Record(record.clone()))] + ); + + // Release restores the legacy fields as they were before the fence. + let release = request( + "db", + OP, + 2, + FenceState::SourceWriteFenced, + 2, + FenceCommand::ReleaseSourceWriteFence, + ); + let commit = store + .apply_fence_command(release, ctx(4_000)) + .await + .unwrap(); + assert_eq!(commit.record.unwrap().revision, 3); + let row = raw_config(&conn, "db"); + assert!(!row.block_writes && !row.block_reads); + assert_eq!(row.block_reason.as_deref(), Some("pre-fence")); + assert_eq!(row.max_db_pages, 1234); + } + + #[tokio::test] + async fn flag_off_still_enforces_existing_fences() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + drop(handle); + drop(store); + + let store = open(dir.path(), false).await; + assert!(!store.fence_enabled()); + assert!(store.fence_enforced()); + let e = fence_error( + store + .apply_fence_command(acquire("db", OTHER_OP, 2), ctx(2_000)) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::FenceDisabled)); + let handle = store.handle("db".into()).await.unwrap(); + let e = fence_error(handle.store(DatabaseConfig::default()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert_eq!( + store + .inspect_fence("db".into()) + .await + .unwrap() + .fence + .state(), + FenceState::SourceDraining + ); + } + + #[tokio::test] + async fn disabled_fence_creates_nothing() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), false).await; + let handle = create_namespace(&store, "db").await; + assert!(!store.fence_enforced()); + let e = fence_error( + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FencePreconditionFailed); + assert_eq!(e.detail(), Some(FenceDetail::FenceDisabled)); + let conn = raw(dir.path()).await; + assert!(!fence_store::tables_exist(&conn).unwrap()); + handle + .store(DatabaseConfig { + max_db_pages: 7, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(raw_config(&conn, "db").max_db_pages, 7); + assert_eq!( + store.inspect_fence("db".into()).await.unwrap().fence, + StoredFence::None { + namespace_exists: true + } + ); + assert!(store.load_fences().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn concurrent_cas_has_exactly_one_winner() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "db").await; + + let attempts = (0..16u128).map(|i| { + let store = store.clone(); + async move { + store + .apply_fence_command(acquire("db", Uuid::from_u128(0x100 + i), 1), ctx(1_000)) + .await + } + }); + let outcomes: Vec<_> = futures::future::join_all(attempts) + .await + .into_iter() + .map(outcome_of) + .collect(); + assert_eq!( + outcomes + .iter() + .filter(|o| **o == FenceOutcome::Draining) + .count(), + 1, + "{outcomes:?}" + ); + assert!(outcomes.iter().all(|o| matches!( + o, + FenceOutcome::Draining | FenceOutcome::FenceOwnedByAnotherOperation + ))); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(inspection.fence.revision(), 1); + assert_eq!(inspection.receipts.len(), 1); + } + + #[tokio::test] + async fn fence_cas_and_config_writes_serialise() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + + // Another metastore connection holds the write lock with an uncommitted config + // change. The fence transition cannot interleave with it: it gives up on the lock + // and writes nothing. + let mut other = raw(dir.path()).await; + let tx = other + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let mut changed = raw_config(&tx, "db"); + changed.max_db_pages = 42; + fence_store::write_config_row(&tx, &"db".into(), &changed).unwrap(); + let r = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await; + assert!( + matches!(r, Err(Error::RusqliteError(_))), + "expected the write lock to be busy, got {r:?}" + ); + tx.commit().unwrap(); + assert_eq!( + store + .inspect_fence("db".into()) + .await + .unwrap() + .fence + .state(), + FenceState::Unfenced + ); + + // After the commit the transition reads the row as committed, so the other writer's + // change survives underneath the mirror, although the in-memory config never saw it. + assert_eq!(handle.get().max_db_pages, 1234); + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "db"); + assert_eq!(row.max_db_pages, 42); + assert!(row.block_writes); + + // An ordinary config write is refused inside its transaction while the fence denies + // lifecycle operations, and a refused write is not published. + let e = fence_error( + handle + .store(DatabaseConfig { + max_db_pages: 9, + ..Default::default() + }) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert_eq!(handle.get().max_db_pages, 1234); + assert_eq!(raw_config(&conn, "db").max_db_pages, 42); + let e = fence_error(handle.flush().await); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + + // So is a delete, and the fence row would stop an older binary's delete too. + let e = fence_error(remove_blocking(&store, "db")); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert!(conn + .execute("DELETE FROM namespace_configs WHERE namespace = 'db'", ()) + .is_err()); + + // Once released, config writes and delete work again; delete takes the fence with it. + let release = request( + "db", + OP, + 2, + FenceState::SourceDraining, + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + store + .apply_fence_command(release, ctx(2_000)) + .await + .unwrap(); + handle + .store(DatabaseConfig { + max_db_pages: 9, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(handle.get().max_db_pages, 9); + drop(handle); + assert!(remove_blocking(&store, "db").unwrap().is_some()); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!( + inspection.fence, + StoredFence::None { + namespace_exists: false + } + ); + assert!(inspection.receipts.is_empty()); + } + + #[tokio::test] + async fn corrupt_fence_row_fails_closed() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + drop(handle); + drop(store); + + let conn = raw(dir.path()).await; + conn.execute( + "UPDATE namespace_fences SET record = x'00ff00' WHERE namespace = 'db'", + (), + ) + .unwrap(); + + // Startup does not fail, and does not guess. + let store = open(dir.path(), true).await; + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + .. + } + )); + // The namespace is not served and cannot be recreated. Its in-memory config keeps the + // stored mirror, so statement-level checks would stay closed too. + assert_eq!( + fence_error(store.lookup(&"db".into()).await).detail(), + Some(FenceDetail::CorruptRecord) + ); + assert_eq!( + fence_error(store.handle("db".into()).await).detail(), + Some(FenceDetail::CorruptRecord) + ); + let config = store.inner.configs.lock().await[&NamespaceName::from("db")] + .borrow() + .config + .clone(); + assert!(config.block_writes); + // A handle taken before the fence became unavailable cannot write the config either. + let handle = MetaStoreHandle { + namespace: "db".into(), + inner: HandleState::External( + store.changes_tx.clone(), + store.inner.configs.lock().await[&NamespaceName::from("db")].subscribe(), + ), + }; + + let release = request( + "db", + OP, + 2, + FenceState::SourceDraining, + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + let e = fence_error(store.apply_fence_command(release, ctx(2_000)).await); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + assert_eq!(e.detail(), Some(FenceDetail::CorruptRecord)); + let e = fence_error(handle.store(DatabaseConfig::default()).await); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + let e = fence_error( + store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }, + ctx(2_000), + ) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + + // An unknown format version is its own reason. + conn.execute( + "UPDATE namespace_fences SET format_version = 99 WHERE namespace = 'db'", + (), + ) + .unwrap(); + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::UnsupportedFormatVersion, + .. + } + )); + } + + #[tokio::test] + async fn marker_tracks_the_metastore() { + let dir = tempdir().unwrap(); + let dbs = dir.path().join("dbs"); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + let conn = raw(dir.path()).await; + let (v1, r1, b1): (i64, i64, Vec) = conn + .query_row( + "SELECT format_version, revision, record FROM namespace_fences", + (), + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + let commit = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 7, + }, + }, + ctx(2_000), + ) + .await + .unwrap(); + let record = commit.record.unwrap(); + drop(handle); + drop(store); + + // A marker lost after the commit is rewritten from the metastore on load. + std::fs::remove_file(fence_store::marker_path(&dbs, &"db".into())).unwrap(); + let store = open(dir.path(), true).await; + let marker = fence_store::read_marker(&dbs, &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + drop(store); + + // A metastore that went backwards (restored to revision 1) is not trusted over the + // newer marker, and loading it does not overwrite the marker. + conn.execute( + "UPDATE namespace_fences SET format_version = ?1, revision = ?2, record = ?3", + rusqlite::params![v1, r1, b1], + ) + .unwrap(); + let store = open(dir.path(), true).await; + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + match fence { + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(m), + .. + } => assert_eq!(m, record), + other => panic!("expected metastore_behind_marker, got {other:?}"), + } + let marker = fence_store::read_marker(&dbs, &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + let e = fence_error( + store + .apply_fence_command( + request( + "db", + OP, + 3, + FenceState::SourceWriteFenced, + 2, + FenceCommand::ReleaseSourceWriteFence, + ), + ctx(3_000), + ) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::MetastoreBehindMarker)); + } + + fn create_target(ns: &'static str, command_id: u128) -> FenceRequest { + request( + ns, + OP, + command_id, + FenceState::Absent, + 0, + FenceCommand::CreateTargetQuarantined { + config: TargetConfig { + max_db_size: Some(4096 * 100), + ..Default::default() + }, + }, + ) + } + + #[tokio::test] + async fn target_creation_is_atomic_and_replayable() { + let dir = tempdir().unwrap(); + let dbs = dir.path().join("dbs"); + let store = open(dir.path(), true).await; + + let commit = store + .apply_fence_command(create_target("tgt", 1), ctx(1_000)) + .await + .unwrap(); + assert_eq!(commit.receipt.outcome, FenceOutcome::Applied); + let record = commit.record.clone().unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::TargetQuarantined, 1) + ); + assert_eq!(record.identity.target_incarnation_id, Some(INCARNATION)); + let created = commit.created_config.unwrap(); + assert_eq!(created.max_db_pages, 100); + assert!(!created.block_reads && !created.block_writes); + // Not published: the caller installs the gate first. + assert!(!store.exists(&"tgt".into()).await); + // Stored with the legacy mirror, and with its marker. + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "tgt"); + assert!(row.block_reads && row.block_writes); + assert_eq!( + fence_store::read_marker(&dbs, &"tgt".into()) + .unwrap() + .unwrap() + .unwrap() + .record, + record + ); + let replay = store + .apply_fence_command(create_target("tgt", 1), ctx(2_000)) + .await + .unwrap(); + assert_eq!(replay.kind, FenceCommitKind::Replayed); + // A new command of the owner asking for the same thing is ALREADY_APPLIED; another + // operation cannot take the name. + let again = store + .apply_fence_command(create_target("tgt", 2), ctx(2_000)) + .await + .unwrap(); + assert_eq!(again.receipt.outcome, FenceOutcome::AlreadyApplied); + assert_eq!(again.record.unwrap().revision, 1); + let mut other = create_target("tgt", 5); + other.operation_id = OTHER_OP; + let e = fence_error(store.apply_fence_command(other, ctx(2_000)).await); + assert_eq!(e.outcome(), FenceOutcome::FenceOwnedByAnotherOperation); + + // A crash between the marker and the commit: the marker is all that is left. + store + .apply_fence_command(create_target("tgt2", 3), ctx(3_000)) + .await + .unwrap(); + let marker = fence_store::read_marker(&dbs, &"tgt2".into()) + .unwrap() + .unwrap() + .unwrap(); + for sql in [ + "DELETE FROM namespace_fence_receipts WHERE namespace = 'tgt2'", + "DELETE FROM namespace_fences WHERE namespace = 'tgt2'", + "DELETE FROM namespace_configs WHERE namespace = 'tgt2'", + ] { + conn.execute(sql, ()).unwrap(); + } + let fence = store.inspect_fence("tgt2".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + .. + } + )); + // Only the same command completes it, keeping the incarnation id it announced. + let e = fence_error( + store + .apply_fence_command(create_target("tgt2", 4), ctx(4_000)) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + let mut later = ctx(5_000); + later.new_incarnation_id = Uuid::from_u128(0x21); + let commit = store + .apply_fence_command(create_target("tgt2", 3), later) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + let record = commit.record.unwrap(); + assert_eq!( + record.identity.target_incarnation_id, + marker.record.identity.target_incarnation_id + ); + assert_eq!( + store.inspect_fence("tgt2".into()).await.unwrap().fence, + StoredFence::Record(record) + ); + } + + async fn run_source_operation(store: &MetaStore, op: Uuid, base: u128, rev: u64, now: i64) { + let expected = if rev == 0 { + FenceState::Unfenced + } else { + FenceState::Released + }; + let mut acq = acquire("db", op, base); + acq.expected_state = expected; + acq.expected_revision = rev; + store.apply_fence_command(acq, ctx(now)).await.unwrap(); + let release = request( + "db", + op, + base + 1, + FenceState::SourceDraining, + rev + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + store + .apply_fence_command(release, ctx(now + 1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn receipts_of_finished_operations_are_pruned_after_retention() { + let dir = tempdir().unwrap(); + let store = open_with( + dir.path(), + MetaStoreConfig { + namespace_fence: true, + namespace_fence_receipt_retention: Some(Duration::from_secs(1)), + ..Default::default() + }, + ) + .await; + let _handle = create_namespace(&store, "db").await; + + run_source_operation(&store, OP, 1, 0, 1_000).await; + // Within the retention period, the finished operation's receipts are kept. + run_source_operation(&store, OTHER_OP, 10, 2, 1_500).await; + let ops = |receipts: &[StoredReceipt]| { + receipts + .iter() + .map(|r| r.receipt.clone().unwrap().operation_id) + .collect::>() + }; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(ops(&inspection.receipts), vec![OP, OP, OTHER_OP, OTHER_OP]); + // Later, a transition prunes other operations' old receipts, never the owner's. + run_source_operation(&store, Uuid::from_u128(0xc), 20, 4, 10_000).await; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!( + ops(&inspection.receipts), + vec![Uuid::from_u128(0xc), Uuid::from_u128(0xc)] + ); + assert_eq!(inspection.fence.revision(), 6); + } + + /// Fail-closed metastore recovery (`docs/NAMESPACE_FENCE.md` section 13.3). + mod recovery { + use super::*; + + fn unavailable_detail(r: Result) -> FenceDetail { + let e = fence_error(r); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable, "{e}"); + e.detail().expect("unavailable carries a detail") + } + + async fn open_err(dir: &Path, config: MetaStoreConfig) -> Error { + let (maker, manager) = metastore_connection_maker(None, dir).await.unwrap(); + let conn = maker().unwrap(); + match MetaStore::new(config, dir, conn, manager, DatabaseKind::Primary).await { + Ok(_) => panic!("the metastore opened"), + Err(e) => e, + } + } + + fn recover_from_fs(fence: bool) -> MetaStoreConfig { + MetaStoreConfig { + allow_recover_from_fs: true, + namespace_fence: fence, + ..Default::default() + } + } + + /// A fenced namespace `db` (SOURCE_DRAINING, revision 1, with its marker). + async fn fenced_db(dir: &Path) -> NamespaceFenceRecord { + let store = open(dir, true).await; + let _handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap() + .record + .unwrap() + } + + fn assert_marker_unavailable(fence: &StoredFence, record: &NamespaceFenceRecord) { + match fence { + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(m), + .. + } => assert_eq!(m, record), + other => panic!("expected metastore_behind_marker, got {other:?}"), + } + } + + #[tokio::test] + async fn lookup_never_creates() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + assert!(store.lookup(&"missing".into()).await.unwrap().is_none()); + assert!(!store.exists(&"missing".into()).await); + + let _handle = create_namespace(&store, "db").await; + let found = store.lookup(&"db".into()).await.unwrap().unwrap(); + assert_eq!(found.get().max_db_pages, 1234); + // Only the creating path adds an entry. + let created = store.handle("new".into()).await.unwrap(); + assert_eq!( + created.get().max_db_pages, + DatabaseConfig::default().max_db_pages + ); + assert!(store.exists(&"new".into()).await); + } + + #[tokio::test] + async fn fs_recovery_with_marker_unavailable() { + for fence in [true, false] { + let dir = tempdir().unwrap(); + let record = fenced_db(dir.path()).await; + // A legacy namespace directory without a marker. + std::fs::create_dir_all(dir.path().join("dbs").join("legacy")).unwrap(); + std::fs::remove_dir_all(dir.path().join("metastore")).unwrap(); + + let store = open_with(dir.path(), recover_from_fs(fence)).await; + // The legacy directory is recovered as before. + let legacy = store.lookup(&"legacy".into()).await.unwrap().unwrap(); + assert_eq!( + legacy.get().max_db_pages, + DatabaseConfig::default().max_db_pages + ); + // The fenced one is not recovered with a guessed config, and is unavailable. + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker, + "fence flag {fence}" + ); + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "db")), + FenceDetail::MetastoreBehindMarker + ); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_marker_unavailable(&inspection.fence, &record); + let fences = store.load_fences().await.unwrap(); + assert_eq!(fences.len(), 1); + assert_marker_unavailable(&fences[0].1, &record); + // Nothing was written for it, and the marker is untouched. + let conn = raw(dir.path()).await; + assert!(fence_store::read_config_row(&conn, &"db".into()) + .unwrap() + .is_none()); + let marker = fence_store::read_marker(&dir.path().join("dbs"), &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + } + } + + #[tokio::test] + async fn destroy_on_error_keeps_fenced_unavailable() { + let dir = tempdir().unwrap(); + let record = fenced_db(dir.path()).await; + { + // Break the metastore so that restoring it fails. + let conn = raw(dir.path()).await; + conn.execute( + "ALTER TABLE namespace_configs RENAME COLUMN config TO broken", + (), + ) + .unwrap(); + } + let store = open_with( + dir.path(), + MetaStoreConfig { + destroy_on_error: true, + namespace_fence: true, + ..Default::default() + }, + ) + .await; + // The broken metastore is kept aside, not deleted. + let aside: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .filter(|n| n.starts_with("metastore.broken-")) + .collect(); + assert_eq!(aside.len(), 1, "{aside:?}"); + assert!(dir.path().join(&aside[0]).join("data").exists()); + // The rebuilt metastore knows nothing of `db`, so its marker makes it unavailable. + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_marker_unavailable( + &store.inspect_fence("db".into()).await.unwrap().fence, + &record, + ); + } + + #[tokio::test] + async fn destroy_on_error_without_fences_is_unchanged() { + let dir = tempdir().unwrap(); + { + let store = open(dir.path(), false).await; + let _handle = create_namespace(&store, "db").await; + } + { + let conn = raw(dir.path()).await; + conn.execute( + "ALTER TABLE namespace_configs RENAME COLUMN config TO broken", + (), + ) + .unwrap(); + } + let store = open_with( + dir.path(), + MetaStoreConfig { + destroy_on_error: true, + ..Default::default() + }, + ) + .await; + assert!(store.lookup(&"db".into()).await.unwrap().is_none()); + assert!(!std::fs::read_dir(dir.path()).unwrap().any(|e| e + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("metastore."))); + } + + #[tokio::test] + async fn undecodable_row_unavailable() { + let dir = tempdir().unwrap(); + { + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "good").await; + } + let conn = raw(dir.path()).await; + conn.execute( + "INSERT INTO namespace_configs VALUES ('bad', X'FFFFFFFF')", + (), + ) + .unwrap(); + let store = open(dir.path(), true).await; + assert!(store.lookup(&"good".into()).await.unwrap().is_some()); + assert_eq!( + unavailable_detail(store.lookup(&"bad".into()).await), + FenceDetail::CorruptRecord + ); + // Never replaced by a default config, nor deleted. + assert_eq!( + unavailable_detail(store.handle("bad".into()).await), + FenceDetail::CorruptRecord + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "bad")), + FenceDetail::CorruptRecord + ); + assert!(matches!( + store.inspect_fence("bad".into()).await.unwrap().fence, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + .. + } + )); + let bytes: Vec = conn + .query_row( + "SELECT config FROM namespace_configs WHERE namespace = 'bad'", + (), + |r| r.get(0), + ) + .unwrap(); + assert_eq!(bytes, vec![0xff; 4]); + } + + #[tokio::test] + async fn undecodable_row_without_fences_is_skipped_as_before() { + let dir = tempdir().unwrap(); + drop(open(dir.path(), false).await); + let conn = raw(dir.path()).await; + conn.execute( + "INSERT INTO namespace_configs VALUES ('bad', X'FFFFFFFF')", + (), + ) + .unwrap(); + let store = open(dir.path(), false).await; + assert!(store.lookup(&"bad".into()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn undecodable_name_with_fence_fails_startup() { + let dir = tempdir().unwrap(); + drop(open(dir.path(), true).await); + let conn = raw(dir.path()).await; + let config = metadata::DatabaseConfig::from(&DatabaseConfig::default()).encode_to_vec(); + conn.execute("INSERT INTO namespace_configs VALUES ('', ?1)", [&config]) + .unwrap(); + // Without a fence it is skipped, as before. + let store = open(dir.path(), true).await; + assert!(store.load_fences().await.unwrap().is_empty()); + drop(store); + conn.execute("INSERT INTO namespace_fences VALUES ('', 1, 1, X'00')", ()) + .unwrap(); + let e = open_err(dir.path(), MetaStoreConfig::default()).await; + assert!(matches!(e, Error::Internal(_)), "{e}"); + } + + #[tokio::test] + async fn marker_in_invalid_directory_fails_startup() { + use std::os::unix::ffi::OsStrExt; + let dir = tempdir().unwrap(); + let bad = dir + .path() + .join("dbs") + .join(std::ffi::OsStr::from_bytes(b"\xff")); + std::fs::create_dir_all(&bad).unwrap(); + std::fs::write(bad.join(fence_store::MARKER_FILE_NAME), b"x").unwrap(); + let e = open_err(dir.path(), MetaStoreConfig::default()).await; + assert!(matches!(e, Error::Internal(_)), "{e}"); + } + + #[tokio::test] + async fn incomplete_target_unavailable() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let commit = store + .apply_fence_command(create_target("tgt", 3), ctx(1_000)) + .await + .unwrap(); + let record = commit.record.unwrap(); + // Committed but not yet published: nothing can create a default namespace over it. + let e = fence_error(store.handle("tgt".into()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationTargetQuarantined); + assert!(store.lookup(&"tgt".into()).await.unwrap().is_none()); + drop(store); + + // A crash between the marker and the commit: the marker is all that is left. + let conn = raw(dir.path()).await; + for sql in [ + "DELETE FROM namespace_fence_receipts WHERE namespace = 'tgt'", + "DELETE FROM namespace_fences WHERE namespace = 'tgt'", + "DELETE FROM namespace_configs WHERE namespace = 'tgt'", + ] { + conn.execute(sql, ()).unwrap(); + } + let store = open(dir.path(), true).await; + assert_eq!( + unavailable_detail(store.lookup(&"tgt".into()).await), + FenceDetail::IncompleteTargetCreation + ); + assert_eq!( + unavailable_detail(store.handle("tgt".into()).await), + FenceDetail::IncompleteTargetCreation + ); + let fences = store.load_fences().await.unwrap(); + assert!(matches!( + fences.as_slice(), + [( + _, + StoredFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + .. + } + )] + )); + // The same command completes the creation, which settles the name. + let commit = store + .apply_fence_command(create_target("tgt", 3), ctx(2_000)) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + assert_eq!(commit.record.as_ref().unwrap().identity, record.identity); + assert!(store.lookup(&"tgt".into()).await.unwrap().is_none()); + let e = fence_error(store.handle("tgt".into()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationTargetQuarantined); + assert_eq!( + store.inspect_fence("tgt".into()).await.unwrap().fence, + StoredFence::Record(commit.record.unwrap()) + ); + } + + #[tokio::test] + async fn metastore_rollback_detected_by_marker() { + let dir = tempdir().unwrap(); + let conn = raw(dir.path()).await; + let record = { + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "db").await; + let record = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap() + .record + .unwrap(); + drop(store); + // A metastore restored from a backup taken before the fence: the namespace is + // unfenced there, and only its marker remembers the fence. + conn.execute("DELETE FROM namespace_fence_receipts", ()) + .unwrap(); + conn.execute("DELETE FROM namespace_fences", ()).unwrap(); + record + }; + let store = open(dir.path(), true).await; + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + // Neither served, nor deleted, nor given a new config. + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "db")), + FenceDetail::MetastoreBehindMarker + ); + assert_marker_unavailable( + &store.inspect_fence("db".into()).await.unwrap().fence, + &record, + ); + // A new operation cannot acquire over the lost fence either. + let e = fence_error( + store + .apply_fence_command(acquire("db", OTHER_OP, 2), ctx(2_000)) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::MetastoreBehindMarker)); + } + } +} diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index ec45b50445..cba4030090 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -20,6 +20,7 @@ pub use self::store::NamespaceStore; pub mod broadcasters; pub(crate) mod configurator; +pub mod fence; pub mod meta_store; mod name; pub mod replication_wal; diff --git a/libsql-server/src/namespace/store.rs b/libsql-server/src/namespace/store.rs index 86e9438ccd..1813ef5187 100644 --- a/libsql-server/src/namespace/store.rs +++ b/libsql-server/src/namespace/store.rs @@ -173,7 +173,7 @@ impl NamespaceStore { ns.destroy().await?; } - let db_config = self.inner.metadata.handle(namespace.clone()).await; + let db_config = self.inner.metadata.handle(namespace.clone()).await?; // destroy on-disk database self.cleanup( &namespace, @@ -240,7 +240,9 @@ impl NamespaceStore { return Err(crate::Error::NamespaceDoesntExist(from.to_string())); } - let from_config = self.inner.metadata.handle(from.clone()).await; + let Some(from_config) = self.inner.metadata.lookup(&from).await? else { + return Err(crate::Error::NamespaceDoesntExist(from.to_string())); + }; let from_entry = self .load_namespace(&from, from_config.clone(), RestoreOption::Latest) .await?; @@ -275,7 +277,7 @@ impl NamespaceStore { should_delete: true, }; - let handle = self.inner.metadata.handle(to.clone()).await; + let handle = self.inner.metadata.handle(to.clone()).await?; handle .store_and_maybe_flush(Some(to_config.into()), false) .await?; @@ -322,13 +324,6 @@ impl NamespaceStore { where Fun: FnOnce(&Namespace) -> R, { - if namespace != NamespaceName::default() - && !self.inner.metadata.exists(&namespace).await - && !self.inner.allow_lazy_creation - { - return Err(Error::NamespaceDoesntExist(namespace.to_string())); - } - let f = { let name = namespace.clone(); move |ns: NamespaceEntry| async move { @@ -341,7 +336,15 @@ impl NamespaceStore { } }; - let handle = self.inner.metadata.handle(namespace.to_owned()).await; + // A lookup that cannot create: only the default namespace and lazy creation create a + // namespace here, and those refuse a name whose fence state is not established. + let handle = match self.inner.metadata.lookup(&namespace).await? { + Some(handle) => handle, + None if namespace == NamespaceName::default() || self.inner.allow_lazy_creation => { + self.inner.metadata.handle(namespace.clone()).await? + } + None => return Err(Error::NamespaceDoesntExist(namespace.to_string())), + }; f(self .load_namespace(&namespace, handle, RestoreOption::Latest) .await?) @@ -440,7 +443,7 @@ impl NamespaceStore { } let db_config = Arc::new(db_config); - let handle = self.inner.metadata.handle(namespace.clone()).await; + let handle = self.inner.metadata.handle(namespace.clone()).await?; tracing::debug!("storing db config"); handle.store(db_config).await?; tracing::debug!("completed storing db config, loading namespace"); diff --git a/libsql-server/src/schema/db.rs b/libsql-server/src/schema/db.rs index ec8dcad840..d0bce10128 100644 --- a/libsql-server/src/schema/db.rs +++ b/libsql-server/src/schema/db.rs @@ -486,6 +486,7 @@ mod test { meta_store .handle(schema.into()) .await + .unwrap() .store(DatabaseConfig { is_shared_schema: true, ..Default::default() @@ -502,6 +503,7 @@ mod test { meta_store .handle(name.into()) .await + .unwrap() .store(DatabaseConfig { shared_schema_name: Some(schema.into()), ..Default::default() @@ -579,6 +581,7 @@ mod test { assert!(meta_store .handle("ns1".into()) .await + .unwrap() .store(DatabaseConfig { shared_schema_name: Some("schema1".into()), ..Default::default() diff --git a/libsql-server/tests/bootstrap.rs b/libsql-server/tests/bootstrap.rs index a464f53288..912015e1f9 100644 --- a/libsql-server/tests/bootstrap.rs +++ b/libsql-server/tests/bootstrap.rs @@ -3,7 +3,7 @@ use std::process::Command; #[test] fn bootstrap() { - let iface_files = &["proto/admin_shell.proto"]; + let iface_files = &["proto/admin_shell.proto", "proto/namespace_fence.proto"]; let dirs = &["proto"]; let out_dir = PathBuf::from(std::env!("CARGO_MANIFEST_DIR"))